diff --git a/pillar/__init__.py b/pillar/__init__.py index 646c5a83..ec39f845 100644 --- a/pillar/__init__.py +++ b/pillar/__init__.py @@ -241,7 +241,7 @@ class PillarServer(Eve): def _config_algolia(self): # Algolia search - if self.config['SEARCH_BACKEND'] != 'algolia': + if 'algolia' not in self.config['SEARCH_BACKENDS']: return from algoliasearch import algoliasearch diff --git a/pillar/api/search/__init__.py b/pillar/api/search/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pillar/api/search/algolia_indexing.py b/pillar/api/search/algolia_indexing.py new file mode 100644 index 00000000..378b2908 --- /dev/null +++ b/pillar/api/search/algolia_indexing.py @@ -0,0 +1,38 @@ +import logging + +from algoliasearch.helpers import AlgoliaException + +log = logging.getLogger(__name__) + + +def push_updated_user(user_to_index: dict): + """Push an update to the Algolia index when a user item is updated""" + + from pillar.api.utils.algolia import index_user_save + + try: + index_user_save(user_to_index) + except AlgoliaException as ex: + log.warning( + 'Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa + user_to_index.get('username'), + user_to_index.get('objectID'), ex) + + +def index_node_save(node_to_index: dict): + from pillar.api.utils import algolia + + try: + algolia.index_node_save(node_to_index) + except AlgoliaException as ex: + log.warning( + 'Unable to push node info to Algolia for node %s; %s', node_to_index, ex) # noqa + + +def index_node_delete(delete_id: str): + + from pillar.api.utils import algolia + try: + algolia.index_node_delete(delete_id) + except AlgoliaException as ex: + log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex) # noqa diff --git a/pillar/api/search/documents.py b/pillar/api/search/documents.py new file mode 100644 index 00000000..4578d1c9 --- /dev/null +++ b/pillar/api/search/documents.py @@ -0,0 +1,66 @@ +import logging + +import elasticsearch_dsl as es +from elasticsearch_dsl import analysis +# from pillar import current_app + +# define elasticsearch document mapping. + + +log = logging.getLogger(__name__) + + +edge_ngram_filter = analysis.token_filter( + 'edge_ngram_filter', + type='edge_ngram', + min_gram=1, + max_gram=15 +) + + +autocomplete = es.analyzer( + 'autocomplete', + tokenizer='standard', + filter=['lowercase', edge_ngram_filter] +) + + +class User(es.DocType): + """ + Elastic document describing user + """ + + name = es.String( + fielddata=True, + analyzer=autocomplete, + ) + + +class Node(es.DocType): + """ + Elastic document describing user + """ + + node_type = es.Keyword() + + x_code = es.String( + multi=True, + fielddata=True, + analyzer=autocomplete, + ) + + +def create_doc_from_user_data(user_to_index): + doc_id = user_to_index['objectID'] + doc = User(_id=doc_id) + return doc + + +def create_doc_from_node_data(node_to_index): + + # node stuff + doc_id = node_to_index['objectID'] + doc = Node(_id=doc_id) + doc.node_type = node_to_index['node_type'] + + return doc diff --git a/pillar/api/search/elastic_indexing.py b/pillar/api/search/elastic_indexing.py new file mode 100644 index 00000000..dce8f58d --- /dev/null +++ b/pillar/api/search/elastic_indexing.py @@ -0,0 +1,25 @@ +import logging + +log = logging.getLogger(__name__) + + +def push_updated_user(user_to_index: dict): + """Push an update to the Algolia index when a user item is updated""" + + log.warning( + 'WIP USER ELK INDEXING %s %s', + user_to_index.get('username'), + user_to_index.get('objectID')) + + +def index_node_save(node_to_index: dict): + + log.warning( + 'WIP USER NODE INDEXING %s', + node_to_index.get('objectID')) + + +def index_node_delete(delete_id: str): + + log.warning( + 'WIP NODE DELETE INDEXING %s', delete_id) diff --git a/pillar/api/utils/algolia.py b/pillar/api/utils/algolia.py index c1daafda..eabc4af1 100644 --- a/pillar/api/utils/algolia.py +++ b/pillar/api/utils/algolia.py @@ -3,123 +3,32 @@ import logging from bson import ObjectId from pillar import current_app -from pillar.api.file_storage import generate_link from . import skip_when_testing log = logging.getLogger(__name__) -INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'} - @skip_when_testing -def algolia_index_user_save(user): +def index_user_save(to_index_user: dict): index_users = current_app.algolia_index_users if not index_users: log.debug('No Algolia index defined, so nothing to do.') return - user_roles = set(user.get('roles', ())) - if 'service' in user_roles: - return - - # Strip unneeded roles - index_roles = user_roles.intersection(current_app.user_roles_indexable) - # Create or update Algolia index for the user - index_users.save_object({ - 'objectID': user['_id'], - 'full_name': user['full_name'], - 'username': user['username'], - 'roles': list(index_roles), - 'groups': user['groups'], - 'email': user['email'] - }) - - log.debug( - 'Pushed user %r to Algolia index %r', - user['_id'], index_users.index_name) - - -def _handle_picture(node: dict, to_index: dict): - """ - add picture fields to be indexed - """ - - if not node.get('picture'): - return - - files_collection = current_app.data.driver.db['files'] - lookup = {'_id': ObjectId(node['picture'])} - picture = files_collection.find_one(lookup) - - img_variation_t = next( - (item for item in picture['variations'] - if item['size'] == 't'), None) - - if img_variation_t: - to_index['picture'] = generate_link( - picture['backend'], - img_variation_t['file_path'], - project_id=str(picture['project']), - is_public=True) + index_users.save_object() @skip_when_testing -def algolia_index_node_save(node): +def index_node_save(node_to_index): + if not current_app.algolia_index_nodes: return - if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES: - return - # If a nodes does not have status published, do not index - if node['properties'].get('status') != 'published': - return - - projects_collection = current_app.data.driver.db['projects'] - project = projects_collection.find_one({'_id': ObjectId(node['project'])}) - - users_collection = current_app.data.driver.db['users'] - user = users_collection.find_one({'_id': ObjectId(node['user'])}) - - to_index = { - 'objectID': node['_id'], - 'name': node['name'], - 'project': { - '_id': project['_id'], - 'name': project['name'] - }, - 'created': node['_created'], - 'updated': node['_updated'], - 'node_type': node['node_type'], - 'user': { - '_id': user['_id'], - 'full_name': user['full_name'] - }, - } - - if 'description' in node and node['description']: - to_index['description'] = node['description'] - - _handle_picture(node, to_index) - - # If the node has world permissions, compute the Free permission - if 'world' in node.get('permissions', {}): - if 'GET' in node['permissions']['world']: - to_index['is_free'] = True - - # Append the media key if the node is of node_type 'asset' - if node['node_type'] == 'asset': - to_index['media'] = node['properties']['content_type'] - - # Add extra properties - for prop in ('tags', 'license_notes'): - if prop in node['properties']: - to_index[prop] = node['properties'][prop] - - current_app.algolia_index_nodes.save_object(to_index) + current_app.algolia_index_nodes.save_object(node_to_index) @skip_when_testing -def algolia_index_node_delete(node): +def index_node_delete(delete_id): if current_app.algolia_index_nodes is None: return - current_app.algolia_index_nodes.delete_object(node['_id']) + current_app.algolia_index_nodes.delete_object(delete_id) diff --git a/pillar/celery/algolia_indexing.py b/pillar/celery/algolia_indexing.py index d947983d..378b2908 100644 --- a/pillar/celery/algolia_indexing.py +++ b/pillar/celery/algolia_indexing.py @@ -1,61 +1,38 @@ import logging from algoliasearch.helpers import AlgoliaException -import bson - -from pillar import current_app log = logging.getLogger(__name__) -def push_updated_user(user_id: str): +def push_updated_user(user_to_index: dict): """Push an update to the Algolia index when a user item is updated""" - from pillar.api.utils.algolia import algolia_index_user_save - - user_oid = bson.ObjectId(user_id) - log.info('Retrieving user %s', user_oid) - users_coll = current_app.db('users') - user = users_coll.find_one({'_id': user_oid}) - if user is None: - log.warning('Unable to find user %s, not updating Algolia.', user_oid) - return + from pillar.api.utils.algolia import index_user_save try: - algolia_index_user_save(user) + index_user_save(user_to_index) except AlgoliaException as ex: - log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa - user.get('username'), user_id, ex) + log.warning( + 'Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa + user_to_index.get('username'), + user_to_index.get('objectID'), ex) -def index_node_save(node_id: str): - from pillar.api.utils.algolia import algolia_index_node_save - - node_oid = bson.ObjectId(node_id) - log.info('Retrieving node %s', node_oid) - - nodes_coll = current_app.db('nodes') - node = nodes_coll.find_one({'_id': node_oid}) - - if node is None: - log.warning('Unable to find node %s, not updating Algolia.', node_id) - return +def index_node_save(node_to_index: dict): + from pillar.api.utils import algolia try: - algolia_index_node_save(node) + algolia.index_node_save(node_to_index) except AlgoliaException as ex: - log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex) # noqa + log.warning( + 'Unable to push node info to Algolia for node %s; %s', node_to_index, ex) # noqa -def index_node_delete(node_id: str): - - from pillar.api.utils.algolia import algolia_index_node_delete - - # Deleting a node takes nothing more than the ID anyway. - # No need to fetch anything from Mongo. - fake_node = {'_id': bson.ObjectId(node_id)} +def index_node_delete(delete_id: str): + from pillar.api.utils import algolia try: - algolia_index_node_delete(fake_node) + algolia.index_node_delete(delete_id) except AlgoliaException as ex: - log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex) # noqa + log.warning('Unable to delete node info to Algolia for node %s; %s', delete_id, ex) # noqa diff --git a/pillar/celery/search_index_tasks.py b/pillar/celery/search_index_tasks.py index ce17c69a..e9bf2412 100644 --- a/pillar/celery/search_index_tasks.py +++ b/pillar/celery/search_index_tasks.py @@ -1,45 +1,179 @@ import logging +from bson import ObjectId from pillar import current_app +from pillar.api.file_storage import generate_link -from . import algolia_indexing -# from . import elastic_indexing +# TODO WIP (stephan) make index backend conditional on settings. +from pillar.api.search import elastic_indexing + +from pillar.api.search import algolia_indexing log = logging.getLogger(__name__) -# TODO(stephan) make index backend conditional on settings. + +INDEX_ALLOWED_NODE_TYPES = {'asset', 'texture', 'group', 'hdri'} + SEARCH_BACKENDS = { 'algolia': algolia_indexing, - 'elastic': None, # elastic_indexing + 'elastic': elastic_indexing } +def _get_node_from_id(node_id: str): + """ + """ + node_oid = ObjectId(node_id) + log.info('Retrieving node %s', node_oid) + + nodes_coll = current_app.db('nodes') + node = nodes_coll.find_one({'_id': node_oid}) + + return node + + +def _handle_picture(node: dict, to_index: dict): + """ + add picture fields to be indexed + """ + + if not node.get('picture'): + return + + files_collection = current_app.data.driver.db['files'] + lookup = {'_id': ObjectId(node['picture'])} + picture = files_collection.find_one(lookup) + + img_variation_t = next( + (item for item in picture['variations'] + if item['size'] == 't'), None) + + if img_variation_t: + to_index['picture'] = generate_link( + picture['backend'], + img_variation_t['file_path'], + project_id=str(picture['project']), + is_public=True) + + +def prepare_node_data(node_id: str): + """ + Given node build data object with fields to index + """ + node = _get_node_from_id(node_id) + + if node is None: + log.warning('Unable to find node %s, not updating Algolia.', node_id) + return + + if node['node_type'] not in INDEX_ALLOWED_NODE_TYPES: + return + # If a nodes does not have status published, do not index + if node['properties'].get('status') != 'published': + return + + projects_collection = current_app.data.driver.db['projects'] + project = projects_collection.find_one({'_id': ObjectId(node['project'])}) + + users_collection = current_app.data.driver.db['users'] + user = users_collection.find_one({'_id': ObjectId(node['user'])}) + + to_index = { + 'objectID': node['_id'], + 'name': node['name'], + 'project': { + '_id': project['_id'], + 'name': project['name'] + }, + 'created': node['_created'], + 'updated': node['_updated'], + 'node_type': node['node_type'], + 'user': { + '_id': user['_id'], + 'full_name': user['full_name'] + }, + } + + if 'description' in node and node['description']: + to_index['description'] = node['description'] + + _handle_picture(node, to_index) + + # If the node has world permissions, compute the Free permission + if 'world' in node.get('permissions', {}): + if 'GET' in node['permissions']['world']: + to_index['is_free'] = True + + # Append the media key if the node is of node_type 'asset' + if node['node_type'] == 'asset': + to_index['media'] = node['properties']['content_type'] + + # Add extra properties + for prop in ('tags', 'license_notes'): + if prop in node['properties']: + to_index[prop] = node['properties'][prop] + + return to_index + + +def prepare_user_data(user_id: str): + """ + Prepare data to index for user node + """ + + user_oid = ObjectId(user_id) + log.info('Retrieving user %s', user_oid) + users_coll = current_app.db('users') + user = users_coll.find_one({'_id': user_oid}) + if user is None: + log.warning('Unable to find user %s, not updating Algolia.', user_oid) + return + + user_roles = set(user.get('roles', ())) + if 'service' in user_roles: + return + + # Strip unneeded roles + index_roles = user_roles.intersection(current_app.user_roles_indexable) + + log.debug('Pushed user %r to Search index', user['_id']) + + user_to_index = { + 'objectID': user['_id'], + 'full_name': user['full_name'], + 'username': user['username'], + 'roles': list(index_roles), + 'groups': user['groups'], + 'email': user['email'] + } + + return user_to_index + + @current_app.celery.task(ignore_result=True) def updated_user(user_id: str): """Push an update to the index when a user item is updated""" - algolia_indexing.push_updated_user(user_id) + user_to_index = prepare_user_data(user_id) + + for searchoption in current_app.config['SEARCH_BACKENDS']: + for searchmodule in SEARCH_BACKENDS[searchoption]: + searchmodule.push_updated_user(user_to_index) @current_app.celery.task(ignore_result=True) def node_save(node_id: str): - algolia_indexing.index_node_save(node_id) + to_index = prepare_node_data(node_id) + + algolia_indexing.index_node_save(to_index) @current_app.celery.task(ignore_result=True) def node_delete(node_id: str): - algolia_indexing.index_node_delete(node_id) - - - -def build_doc_to_index_from(node: dict): - """ - Given node build an to_index document - """ - pass - - - + # Deleting a node takes nothing more than the ID anyway. + # No need to fetch anything from Mongo. + delete_id = ObjectId(node_id) + algolia_indexing.index_node_delete(delete_id) diff --git a/pillar/config.py b/pillar/config.py index 89bee431..deb60871 100644 --- a/pillar/config.py +++ b/pillar/config.py @@ -73,7 +73,7 @@ ALGOLIA_API_KEY = '-SECRET-' ALGOLIA_INDEX_USERS = 'dev_Users' ALGOLIA_INDEX_NODES = 'dev_Nodes' -SEARCH_BACKEND = 'algolia' # algolia, elastic +SEARCH_BACKENDS = ['algolia', 'elastic'] ZENCODER_API_KEY = '-SECRET-'