From fcf19de786a61ef02d6d6858630403047edd5ef7 Mon Sep 17 00:00:00 2001 From: Stephan Preeker Date: Fri, 3 Nov 2017 16:40:02 +0100 Subject: [PATCH] T53161 proces feedback sybren, replace angolia with search --- pillar/api/users/__init__.py | 10 ++-- pillar/api/users/hooks.py | 8 ++-- pillar/api/utils/algolia.py | 46 ++++++++++--------- .../{algolia_tasks.py => algolia_indexing.py} | 3 +- pillar/celery/search_index_tasks.py | 31 +++++++++---- pillar/config.py | 1 + src/scripts/algolia_search.js | 2 + 7 files changed, 61 insertions(+), 40 deletions(-) rename pillar/celery/{algolia_tasks.py => algolia_indexing.py} (97%) diff --git a/pillar/api/users/__init__.py b/pillar/api/users/__init__.py index 664f78c0..20329c3a 100644 --- a/pillar/api/users/__init__.py +++ b/pillar/api/users/__init__.py @@ -31,7 +31,7 @@ def add_user_to_group(user_id: bson.ObjectId, group_id: bson.ObjectId): def user_group_action(user_id: bson.ObjectId, group_id: bson.ObjectId, action: str): """Performs a group action (add/remove). - + :param user_id: the user's ObjectID. :param group_id: the group's ObjectID. :param action: either '$pull' to remove from a group, or '$addToSet' to add to a group. @@ -54,9 +54,9 @@ def user_group_action(user_id: bson.ObjectId, group_id: bson.ObjectId, action: s f'user not found.') -def _update_algolia_user_changed_role(sender, user: dict): +def _update_search_user_changed_role(sender, user: dict): log.debug('Sending updated user %s to Algolia due to role change', user['_id']) - hooks.push_updated_user_to_algolia(user, original=None) + hooks.push_updated_user_to_search(user, original=None) def setup_app(app, api_prefix): @@ -66,7 +66,7 @@ def setup_app(app, api_prefix): app.on_post_GET_users += hooks.post_GET_user app.on_pre_PUT_users += hooks.check_put_access app.on_pre_PUT_users += hooks.before_replacing_user - app.on_replaced_users += hooks.push_updated_user_to_algolia + app.on_replaced_users += hooks.push_updated_user_to_search app.on_replaced_users += hooks.send_blinker_signal_roles_changed app.on_fetched_item_users += hooks.after_fetching_user app.on_fetched_resource_users += hooks.after_fetching_user_resource @@ -76,4 +76,4 @@ def setup_app(app, api_prefix): app.register_api_blueprint(blueprint_api, url_prefix=api_prefix) - service.signal_user_changed_role.connect(_update_algolia_user_changed_role) + service.signal_user_changed_role.connect(_update_search_user_changed_role) diff --git a/pillar/api/users/hooks.py b/pillar/api/users/hooks.py index 7bdf75d4..9d8033a2 100644 --- a/pillar/api/users/hooks.py +++ b/pillar/api/users/hooks.py @@ -65,15 +65,15 @@ def before_replacing_user(request, lookup): 'email field must be given') -def push_updated_user_to_algolia(user, original): +def push_updated_user_to_search(user, original): """ - Push an update to the Algolia index when a user + Push an update to the Search index when a user item is updated """ - from pillar.celery import search_index_tasks as index + from pillar.celery import search_index_tasks as searchindex - index.updated_user.delay(str(user['_id'])) + searchindex.updated_user.delay(str(user['_id'])) def send_blinker_signal_roles_changed(user, original): diff --git a/pillar/api/utils/algolia.py b/pillar/api/utils/algolia.py index f5e8abcc..c1daafda 100644 --- a/pillar/api/utils/algolia.py +++ b/pillar/api/utils/algolia.py @@ -40,26 +40,28 @@ def algolia_index_user_save(user): user['_id'], index_users.index_name) -def _handle_picture(node, doc): +def _handle_picture(node: dict, to_index: dict): """ add picture fields to be indexed """ - if 'picture' in node and node['picture']: - files_collection = current_app.data.driver.db['files'] - lookup = {'_id': ObjectId(node['picture'])} - picture = files_collection.find_one(lookup) + if not node.get('picture'): + return - img_variation_t = next( - (item for item in picture['variations'] - if item['size'] == 't'), None) + files_collection = current_app.data.driver.db['files'] + lookup = {'_id': ObjectId(node['picture'])} + picture = files_collection.find_one(lookup) - if img_variation_t: - doc['picture'] = generate_link( - picture['backend'], - img_variation_t['file_path'], - project_id=str(picture['project']), - is_public=True) + img_variation_t = next( + (item for item in picture['variations'] + if item['size'] == 't'), None) + + if img_variation_t: + to_index['picture'] = generate_link( + picture['backend'], + img_variation_t['file_path'], + project_id=str(picture['project']), + is_public=True) @skip_when_testing @@ -78,7 +80,7 @@ def algolia_index_node_save(node): users_collection = current_app.data.driver.db['users'] user = users_collection.find_one({'_id': ObjectId(node['user'])}) - doc = { + to_index = { 'objectID': node['_id'], 'name': node['name'], 'project': { @@ -95,25 +97,25 @@ def algolia_index_node_save(node): } if 'description' in node and node['description']: - doc['description'] = node['description'] + to_index['description'] = node['description'] - _handle_picture(node, doc) + _handle_picture(node, to_index) # If the node has world permissions, compute the Free permission - if 'permissions' in node and 'world' in node['permissions']: + if 'world' in node.get('permissions', {}): if 'GET' in node['permissions']['world']: - doc['is_free'] = True + to_index['is_free'] = True # Append the media key if the node is of node_type 'asset' if node['node_type'] == 'asset': - doc['media'] = node['properties']['content_type'] + to_index['media'] = node['properties']['content_type'] # Add extra properties for prop in ('tags', 'license_notes'): if prop in node['properties']: - doc[prop] = node['properties'][prop] + to_index[prop] = node['properties'][prop] - current_app.algolia_index_nodes.save_object(doc) + current_app.algolia_index_nodes.save_object(to_index) @skip_when_testing diff --git a/pillar/celery/algolia_tasks.py b/pillar/celery/algolia_indexing.py similarity index 97% rename from pillar/celery/algolia_tasks.py rename to pillar/celery/algolia_indexing.py index 5218e8f0..d947983d 100644 --- a/pillar/celery/algolia_tasks.py +++ b/pillar/celery/algolia_indexing.py @@ -8,7 +8,7 @@ from pillar import current_app log = logging.getLogger(__name__) -def push_updated_user_to_algolia(user_id: str): +def push_updated_user(user_id: str): """Push an update to the Algolia index when a user item is updated""" from pillar.api.utils.algolia import algolia_index_user_save @@ -48,6 +48,7 @@ def index_node_save(node_id: str): def index_node_delete(node_id: str): + from pillar.api.utils.algolia import algolia_index_node_delete # Deleting a node takes nothing more than the ID anyway. diff --git a/pillar/celery/search_index_tasks.py b/pillar/celery/search_index_tasks.py index 769e331f..ce17c69a 100644 --- a/pillar/celery/search_index_tasks.py +++ b/pillar/celery/search_index_tasks.py @@ -1,30 +1,45 @@ import logging - -from . import algolia_tasks - from pillar import current_app +from . import algolia_indexing +# from . import elastic_indexing + + log = logging.getLogger(__name__) +# TODO(stephan) make index backend conditional on settings. -# TODO make index backend conditional on settings. -# now uses angolia, but should use elastic +SEARCH_BACKENDS = { + 'algolia': algolia_indexing, + 'elastic': None, # elastic_indexing +} @current_app.celery.task(ignore_result=True) def updated_user(user_id: str): """Push an update to the index when a user item is updated""" - algolia_tasks.push_updated_user_to_algolia(user_id) + algolia_indexing.push_updated_user(user_id) @current_app.celery.task(ignore_result=True) def node_save(node_id: str): - algolia_tasks.index_node_save(node_id) + algolia_indexing.index_node_save(node_id) @current_app.celery.task(ignore_result=True) def node_delete(node_id: str): - algolia_tasks.index_node_delete(node_id) + algolia_indexing.index_node_delete(node_id) + + + +def build_doc_to_index_from(node: dict): + """ + Given node build an to_index document + """ + pass + + + diff --git a/pillar/config.py b/pillar/config.py index 526ae4a6..89bee431 100644 --- a/pillar/config.py +++ b/pillar/config.py @@ -75,6 +75,7 @@ ALGOLIA_INDEX_NODES = 'dev_Nodes' SEARCH_BACKEND = 'algolia' # algolia, elastic + ZENCODER_API_KEY = '-SECRET-' ZENCODER_NOTIFICATIONS_SECRET = '-SECRET-' ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/' diff --git a/src/scripts/algolia_search.js b/src/scripts/algolia_search.js index d1b445c6..be2982c1 100644 --- a/src/scripts/algolia_search.js +++ b/src/scripts/algolia_search.js @@ -2,6 +2,8 @@ $(document).ready(function() { /******************** * INITIALIZATION + * + * TODO (stephan) * *******************/ var HITS_PER_PAGE = 25;