T53161 WIP create elasticsearch app / doc / stuff

pillar/api/search/algolia_indexing.py

@@ -0,0 +1,38 @@
+import logging
+
+from algoliasearch.helpers import AlgoliaException
+
+log = logging.getLogger(__name__)
+
+
+def push_updated_user(user_to_index: dict):
+    """Push an update to the Algolia index when a user item is updated"""
+
+    from pillar.api.utils.algolia import index_user_save
+
+    try:
+        index_user_save(user_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
+            user_to_index.get('username'),
+            user_to_index.get('objectID'), ex)
+
+
+def index_node_save(node_to_index: dict):
+    from pillar.api.utils import algolia
+
+    try:
+        algolia.index_node_save(node_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push node info to Algolia for node %s; %s', node_to_index, ex)  # noqa
+
+
+def index_node_delete(delete_id: str):
+
+    from pillar.api.utils import algolia
+    try:
+        algolia.index_node_delete(delete_id)
+    except AlgoliaException as ex:
+        log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex)  # noqa

pillar/api/search/documents.py

@@ -0,0 +1,66 @@
+import logging
+
+import elasticsearch_dsl as es
+from elasticsearch_dsl import analysis
+# from pillar import current_app
+
+# define elasticsearch document mapping.
+
+
+log = logging.getLogger(__name__)
+
+
+edge_ngram_filter = analysis.token_filter(
+    'edge_ngram_filter',
+    type='edge_ngram',
+    min_gram=1,
+    max_gram=15
+)
+
+
+autocomplete = es.analyzer(
+    'autocomplete',
+    tokenizer='standard',
+    filter=['lowercase', edge_ngram_filter]
+)
+
+
+class User(es.DocType):
+    """
+    Elastic document describing user
+    """
+
+    name = es.String(
+        fielddata=True,
+        analyzer=autocomplete,
+    )
+
+
+class Node(es.DocType):
+    """
+    Elastic document describing user
+    """
+
+    node_type = es.Keyword()
+
+    x_code = es.String(
+        multi=True,
+        fielddata=True,
+        analyzer=autocomplete,
+    )
+
+
+def create_doc_from_user_data(user_to_index):
+    doc_id = user_to_index['objectID']
+    doc = User(_id=doc_id)
+    return doc
+
+
+def create_doc_from_node_data(node_to_index):
+
+    # node stuff
+    doc_id = node_to_index['objectID']
+    doc = Node(_id=doc_id)
+    doc.node_type = node_to_index['node_type']
+
+    return doc

pillar/api/search/elastic_indexing.py

@@ -0,0 +1,25 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def push_updated_user(user_to_index: dict):
+    """Push an update to the Algolia index when a user item is updated"""
+
+    log.warning(
+        'WIP USER ELK INDEXING %s %s',
+        user_to_index.get('username'),
+        user_to_index.get('objectID'))
+
+
+def index_node_save(node_to_index: dict):
+
+    log.warning(
+        'WIP USER NODE INDEXING %s',
+        node_to_index.get('objectID'))
+
+
+def index_node_delete(delete_id: str):
+
+    log.warning(
+        'WIP NODE DELETE INDEXING %s', delete_id)

pillar/api/utils/algolia.py

@@ -3,123 +3,32 @@ import logging
 from bson import ObjectId
 
 from pillar import current_app
-from pillar.api.file_storage import generate_link
 from . import skip_when_testing
 
 log = logging.getLogger(__name__)
 
-INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
-
 
 @skip_when_testing
-def algolia_index_user_save(user):
+def index_user_save(to_index_user: dict):
     index_users = current_app.algolia_index_users
     if not index_users:
         log.debug('No Algolia index defined, so nothing to do.')
         return
 
-    user_roles = set(user.get('roles', ()))
-    if 'service' in user_roles:
-        return
-
-    # Strip unneeded roles
-    index_roles = user_roles.intersection(current_app.user_roles_indexable)
-
     # Create or update Algolia index for the user
-    index_users.save_object({
-        'objectID': user['_id'],
-        'full_name': user['full_name'],
-        'username': user['username'],
-        'roles': list(index_roles),
-        'groups': user['groups'],
-        'email': user['email']
-    })
-
-    log.debug(
-        'Pushed user %r to Algolia index %r',
-        user['_id'], index_users.index_name)
-
-
-def _handle_picture(node: dict, to_index: dict):
-    """
-    add picture fields to be indexed
-    """
-
-    if not node.get('picture'):
-        return
-
-    files_collection = current_app.data.driver.db['files']
-    lookup = {'_id': ObjectId(node['picture'])}
-    picture = files_collection.find_one(lookup)
-
-    img_variation_t = next(
-        (item for item in picture['variations']
-         if item['size'] == 't'), None)
-
-    if img_variation_t:
-        to_index['picture'] = generate_link(
-            picture['backend'],
-            img_variation_t['file_path'],
-            project_id=str(picture['project']),
-            is_public=True)
+    index_users.save_object()
 
 
 @skip_when_testing
-def algolia_index_node_save(node):
+def index_node_save(node_to_index):
+
     if not current_app.algolia_index_nodes:
         return
-    if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
-        return
-    # If a nodes does not have status published, do not index
-    if node['properties'].get('status') != 'published':
-        return
-
-    projects_collection = current_app.data.driver.db['projects']
-    project = projects_collection.find_one({'_id': ObjectId(node['project'])})
-
-    users_collection = current_app.data.driver.db['users']
-    user = users_collection.find_one({'_id': ObjectId(node['user'])})
-
-    to_index = {
-        'objectID': node['_id'],
-        'name': node['name'],
-        'project': {
-            '_id': project['_id'],
-            'name': project['name']
-        },
-        'created': node['_created'],
-        'updated': node['_updated'],
-        'node_type': node['node_type'],
-        'user': {
-            '_id': user['_id'],
-            'full_name': user['full_name']
-        },
-    }
-
-    if 'description' in node and node['description']:
-        to_index['description'] = node['description']
-
-    _handle_picture(node, to_index)
-
-    # If the node has world permissions, compute the Free permission
-    if 'world' in node.get('permissions', {}):
-        if 'GET' in node['permissions']['world']:
-            to_index['is_free'] = True
-
-    # Append the media key if the node is of node_type 'asset'
-    if node['node_type'] == 'asset':
-        to_index['media'] = node['properties']['content_type']
-
-    # Add extra properties
-    for prop in ('tags', 'license_notes'):
-        if prop in node['properties']:
-            to_index[prop] = node['properties'][prop]
-
-    current_app.algolia_index_nodes.save_object(to_index)
+    current_app.algolia_index_nodes.save_object(node_to_index)
 
 
 @skip_when_testing
-def algolia_index_node_delete(node):
+def index_node_delete(delete_id):
     if current_app.algolia_index_nodes is None:
         return
-    current_app.algolia_index_nodes.delete_object(node['_id'])
+    current_app.algolia_index_nodes.delete_object(delete_id)