T53161 WIP create elasticsearch app / doc / stuff

pillar/__init__.py

@@ -241,7 +241,7 @@ class PillarServer(Eve):
 
     def _config_algolia(self):
         # Algolia search
-        if self.config['SEARCH_BACKEND'] != 'algolia':
+        if 'algolia' not in self.config['SEARCH_BACKENDS']:
             return
 
         from algoliasearch import algoliasearch

pillar/api/search/algolia_indexing.py

@@ -0,0 +1,38 @@
+import logging
+
+from algoliasearch.helpers import AlgoliaException
+
+log = logging.getLogger(__name__)
+
+
+def push_updated_user(user_to_index: dict):
+    """Push an update to the Algolia index when a user item is updated"""
+
+    from pillar.api.utils.algolia import index_user_save
+
+    try:
+        index_user_save(user_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
+            user_to_index.get('username'),
+            user_to_index.get('objectID'), ex)
+
+
+def index_node_save(node_to_index: dict):
+    from pillar.api.utils import algolia
+
+    try:
+        algolia.index_node_save(node_to_index)
+    except AlgoliaException as ex:
+        log.warning(
+            'Unable to push node info to Algolia for node %s; %s', node_to_index, ex)  # noqa
+
+
+def index_node_delete(delete_id: str):
+
+    from pillar.api.utils import algolia
+    try:
+        algolia.index_node_delete(delete_id)
+    except AlgoliaException as ex:
+        log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex)  # noqa

pillar/api/search/documents.py

@@ -0,0 +1,66 @@
+import logging
+
+import elasticsearch_dsl as es
+from elasticsearch_dsl import analysis
+# from pillar import current_app
+
+# define elasticsearch document mapping.
+
+
+log = logging.getLogger(__name__)
+
+
+edge_ngram_filter = analysis.token_filter(
+    'edge_ngram_filter',
+    type='edge_ngram',
+    min_gram=1,
+    max_gram=15
+)
+
+
+autocomplete = es.analyzer(
+    'autocomplete',
+    tokenizer='standard',
+    filter=['lowercase', edge_ngram_filter]
+)
+
+
+class User(es.DocType):
+    """
+    Elastic document describing user
+    """
+
+    name = es.String(
+        fielddata=True,
+        analyzer=autocomplete,
+    )
+
+
+class Node(es.DocType):
+    """
+    Elastic document describing user
+    """
+
+    node_type = es.Keyword()
+
+    x_code = es.String(
+        multi=True,
+        fielddata=True,
+        analyzer=autocomplete,
+    )
+
+
+def create_doc_from_user_data(user_to_index):
+    doc_id = user_to_index['objectID']
+    doc = User(_id=doc_id)
+    return doc
+
+
+def create_doc_from_node_data(node_to_index):
+
+    # node stuff
+    doc_id = node_to_index['objectID']
+    doc = Node(_id=doc_id)
+    doc.node_type = node_to_index['node_type']
+
+    return doc

pillar/api/search/elastic_indexing.py

@@ -0,0 +1,25 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def push_updated_user(user_to_index: dict):
+    """Push an update to the Algolia index when a user item is updated"""
+
+    log.warning(
+        'WIP USER ELK INDEXING %s %s',
+        user_to_index.get('username'),
+        user_to_index.get('objectID'))
+
+
+def index_node_save(node_to_index: dict):
+
+    log.warning(
+        'WIP USER NODE INDEXING %s',
+        node_to_index.get('objectID'))
+
+
+def index_node_delete(delete_id: str):
+
+    log.warning(
+        'WIP NODE DELETE INDEXING %s', delete_id)

pillar/api/utils/algolia.py

@@ -3,123 +3,32 @@ import logging
 from bson import ObjectId
 
 from pillar import current_app
-from pillar.api.file_storage import generate_link
 from . import skip_when_testing
 
 log = logging.getLogger(__name__)
 
-INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
-
 
 @skip_when_testing
-def algolia_index_user_save(user):
+def index_user_save(to_index_user: dict):
     index_users = current_app.algolia_index_users
     if not index_users:
         log.debug('No Algolia index defined, so nothing to do.')
         return
 
-    user_roles = set(user.get('roles', ()))
-    if 'service' in user_roles:
-        return
-
-    # Strip unneeded roles
-    index_roles = user_roles.intersection(current_app.user_roles_indexable)
-
     # Create or update Algolia index for the user
-    index_users.save_object({
+    index_users.save_object()
-        'objectID': user['_id'],
-        'full_name': user['full_name'],
-        'username': user['username'],
-        'roles': list(index_roles),
-        'groups': user['groups'],
-        'email': user['email']
-    })
-
-    log.debug(
-        'Pushed user %r to Algolia index %r',
-        user['_id'], index_users.index_name)
-
-
-def _handle_picture(node: dict, to_index: dict):
-    """
-    add picture fields to be indexed
-    """
-
-    if not node.get('picture'):
-        return
-
-    files_collection = current_app.data.driver.db['files']
-    lookup = {'_id': ObjectId(node['picture'])}
-    picture = files_collection.find_one(lookup)
-
-    img_variation_t = next(
-        (item for item in picture['variations']
-         if item['size'] == 't'), None)
-
-    if img_variation_t:
-        to_index['picture'] = generate_link(
-            picture['backend'],
-            img_variation_t['file_path'],
-            project_id=str(picture['project']),
-            is_public=True)
 
 
 @skip_when_testing
-def algolia_index_node_save(node):
+def index_node_save(node_to_index):
+
     if not current_app.algolia_index_nodes:
         return
-    if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
+    current_app.algolia_index_nodes.save_object(node_to_index)
-        return
-    # If a nodes does not have status published, do not index
-    if node['properties'].get('status') != 'published':
-        return
-
-    projects_collection = current_app.data.driver.db['projects']
-    project = projects_collection.find_one({'_id': ObjectId(node['project'])})
-
-    users_collection = current_app.data.driver.db['users']
-    user = users_collection.find_one({'_id': ObjectId(node['user'])})
-
-    to_index = {
-        'objectID': node['_id'],
-        'name': node['name'],
-        'project': {
-            '_id': project['_id'],
-            'name': project['name']
-        },
-        'created': node['_created'],
-        'updated': node['_updated'],
-        'node_type': node['node_type'],
-        'user': {
-            '_id': user['_id'],
-            'full_name': user['full_name']
-        },
-    }
-
-    if 'description' in node and node['description']:
-        to_index['description'] = node['description']
-
-    _handle_picture(node, to_index)
-
-    # If the node has world permissions, compute the Free permission
-    if 'world' in node.get('permissions', {}):
-        if 'GET' in node['permissions']['world']:
-            to_index['is_free'] = True
-
-    # Append the media key if the node is of node_type 'asset'
-    if node['node_type'] == 'asset':
-        to_index['media'] = node['properties']['content_type']
-
-    # Add extra properties
-    for prop in ('tags', 'license_notes'):
-        if prop in node['properties']:
-            to_index[prop] = node['properties'][prop]
-
-    current_app.algolia_index_nodes.save_object(to_index)
 
 
 @skip_when_testing
-def algolia_index_node_delete(node):
+def index_node_delete(delete_id):
     if current_app.algolia_index_nodes is None:
         return
-    current_app.algolia_index_nodes.delete_object(node['_id'])
+    current_app.algolia_index_nodes.delete_object(delete_id)

pillar/celery/algolia_indexing.py

@@ -1,61 +1,38 @@
 import logging
 
 from algoliasearch.helpers import AlgoliaException
-import bson
-
-from pillar import current_app
 
 log = logging.getLogger(__name__)
 
 
-def push_updated_user(user_id: str):
+def push_updated_user(user_to_index: dict):
     """Push an update to the Algolia index when a user item is updated"""
 
-    from pillar.api.utils.algolia import algolia_index_user_save
+    from pillar.api.utils.algolia import index_user_save
-
-    user_oid = bson.ObjectId(user_id)
-    log.info('Retrieving user %s', user_oid)
-    users_coll = current_app.db('users')
-    user = users_coll.find_one({'_id': user_oid})
-    if user is None:
-        log.warning('Unable to find user %s, not updating Algolia.', user_oid)
-        return
 
     try:
-        algolia_index_user_save(user)
+        index_user_save(user_to_index)
     except AlgoliaException as ex:
-        log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
+        log.warning(
-                    user.get('username'), user_id, ex)
+            'Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
+            user_to_index.get('username'),
+            user_to_index.get('objectID'), ex)
 
 
-def index_node_save(node_id: str):
+def index_node_save(node_to_index: dict):
-    from pillar.api.utils.algolia import algolia_index_node_save
+    from pillar.api.utils import algolia
-
-    node_oid = bson.ObjectId(node_id)
-    log.info('Retrieving node %s', node_oid)
-
-    nodes_coll = current_app.db('nodes')
-    node = nodes_coll.find_one({'_id': node_oid})
-
-    if node is None:
-        log.warning('Unable to find node %s, not updating Algolia.', node_id)
-        return
 
     try:
-        algolia_index_node_save(node)
+        algolia.index_node_save(node_to_index)
     except AlgoliaException as ex:
-        log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex)  # noqa
+        log.warning(
+            'Unable to push node info to Algolia for node %s; %s', node_to_index, ex)  # noqa
 
 
-def index_node_delete(node_id: str):
+def index_node_delete(delete_id: str):
-
-    from pillar.api.utils.algolia import algolia_index_node_delete
-
-    # Deleting a node takes nothing more than the ID anyway.
-    # No need to fetch anything from Mongo.
-    fake_node = {'_id': bson.ObjectId(node_id)}
 
+    from pillar.api.utils import algolia
     try:
-        algolia_index_node_delete(fake_node)
+        algolia.index_node_delete(delete_id)
     except AlgoliaException as ex:
-        log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex)  # noqa
+        log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex)  # noqa

pillar/celery/search_index_tasks.py

@@ -1,45 +1,179 @@
 import logging
+from bson import ObjectId
 from pillar import current_app
+from pillar.api.file_storage import generate_link
 
-from . import algolia_indexing
+# TODO WIP (stephan) make index backend conditional on settings.
-# from . import elastic_indexing
+from pillar.api.search import elastic_indexing
+
+from pillar.api.search import algolia_indexing
 
 
 log = logging.getLogger(__name__)
 
-# TODO(stephan) make index backend conditional on settings.
+
+INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'}
+
 
 SEARCH_BACKENDS = {
     'algolia': algolia_indexing,
-    'elastic': None,  # elastic_indexing
+    'elastic': elastic_indexing
 }
 
 
+def _get_node_from_id(node_id: str):
+    """
+    """
+    node_oid = ObjectId(node_id)
+    log.info('Retrieving node %s', node_oid)
+
+    nodes_coll = current_app.db('nodes')
+    node = nodes_coll.find_one({'_id': node_oid})
+
+    return node
+
+
+def _handle_picture(node: dict, to_index: dict):
+    """
+    add picture fields to be indexed
+    """
+
+    if not node.get('picture'):
+        return
+
+    files_collection = current_app.data.driver.db['files']
+    lookup = {'_id': ObjectId(node['picture'])}
+    picture = files_collection.find_one(lookup)
+
+    img_variation_t = next(
+        (item for item in picture['variations']
+         if item['size'] == 't'), None)
+
+    if img_variation_t:
+        to_index['picture'] = generate_link(
+            picture['backend'],
+            img_variation_t['file_path'],
+            project_id=str(picture['project']),
+            is_public=True)
+
+
+def prepare_node_data(node_id: str):
+    """
+    Given node build data object with fields to index
+    """
+    node = _get_node_from_id(node_id)
+
+    if node is None:
+        log.warning('Unable to find node %s, not updating Algolia.', node_id)
+        return
+
+    if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES:
+        return
+    # If a nodes does not have status published, do not index
+    if node['properties'].get('status') != 'published':
+        return
+
+    projects_collection = current_app.data.driver.db['projects']
+    project = projects_collection.find_one({'_id': ObjectId(node['project'])})
+
+    users_collection = current_app.data.driver.db['users']
+    user = users_collection.find_one({'_id': ObjectId(node['user'])})
+
+    to_index = {
+        'objectID': node['_id'],
+        'name': node['name'],
+        'project': {
+            '_id': project['_id'],
+            'name': project['name']
+        },
+        'created': node['_created'],
+        'updated': node['_updated'],
+        'node_type': node['node_type'],
+        'user': {
+            '_id': user['_id'],
+            'full_name': user['full_name']
+        },
+    }
+
+    if 'description' in node and node['description']:
+        to_index['description'] = node['description']
+
+    _handle_picture(node, to_index)
+
+    # If the node has world permissions, compute the Free permission
+    if 'world' in node.get('permissions', {}):
+        if 'GET' in node['permissions']['world']:
+            to_index['is_free'] = True
+
+    # Append the media key if the node is of node_type 'asset'
+    if node['node_type'] == 'asset':
+        to_index['media'] = node['properties']['content_type']
+
+    # Add extra properties
+    for prop in ('tags', 'license_notes'):
+        if prop in node['properties']:
+            to_index[prop] = node['properties'][prop]
+
+    return to_index
+
+
+def prepare_user_data(user_id: str):
+    """
+    Prepare data to index for user node
+    """
+
+    user_oid = ObjectId(user_id)
+    log.info('Retrieving user %s', user_oid)
+    users_coll = current_app.db('users')
+    user = users_coll.find_one({'_id': user_oid})
+    if user is None:
+        log.warning('Unable to find user %s, not updating Algolia.', user_oid)
+        return
+
+    user_roles = set(user.get('roles', ()))
+    if 'service' in user_roles:
+        return
+
+    # Strip unneeded roles
+    index_roles = user_roles.intersection(current_app.user_roles_indexable)
+
+    log.debug('Pushed user %r to Search index', user['_id'])
+
+    user_to_index = {
+        'objectID': user['_id'],
+        'full_name': user['full_name'],
+        'username': user['username'],
+        'roles': list(index_roles),
+        'groups': user['groups'],
+        'email': user['email']
+    }
+
+    return user_to_index
+
+
 @current_app.celery.task(ignore_result=True)
 def updated_user(user_id: str):
     """Push an update to the index when a user item is updated"""
 
-    algolia_indexing.push_updated_user(user_id)
+    user_to_index = prepare_user_data(user_id)
+
+    for searchoption in current_app.config['SEARCH_BACKENDS']:
+        for searchmodule in SEARCH_BACKENDS[searchoption]:
+            searchmodule.push_updated_user(user_to_index)
 
 
 @current_app.celery.task(ignore_result=True)
 def node_save(node_id: str):
 
-    algolia_indexing.index_node_save(node_id)
+    to_index = prepare_node_data(node_id)
+
+    algolia_indexing.index_node_save(to_index)
 
 
 @current_app.celery.task(ignore_result=True)
 def node_delete(node_id: str):
 
-    algolia_indexing.index_node_delete(node_id)
+    # Deleting a node takes nothing more than the ID anyway.
-
+    # No need to fetch anything from Mongo.
-
+    delete_id = ObjectId(node_id)
-
+    algolia_indexing.index_node_delete(delete_id)
-def build_doc_to_index_from(node: dict):
-    """
-    Given node build an to_index document
-    """
-    pass
-
-
-

pillar/config.py

@@ -73,7 +73,7 @@ ALGOLIA_API_KEY = '-SECRET-'
 ALGOLIA_INDEX_USERS = 'dev_Users'
 ALGOLIA_INDEX_NODES = 'dev_Nodes'
 
-SEARCH_BACKEND = 'algolia'  # algolia, elastic
+SEARCH_BACKENDS = ['algolia', 'elastic']
 
 
 ZENCODER_API_KEY = '-SECRET-'