From 47a1db07dcfecf624df479bf7f96a6668b5ae0cf Mon Sep 17 00:00:00 2001 From: Stephan Preeker Date: Wed, 25 Oct 2017 17:09:10 +0200 Subject: [PATCH] T53161 start working on elastic.. --- pillar/__init__.py | 2 +- pillar/api/nodes/__init__.py | 14 ++++---- pillar/api/users/hooks.py | 17 ++++++--- pillar/api/utils/algolia.py | 54 +++++++++++++++++++---------- pillar/celery/algolia_tasks.py | 16 ++++----- pillar/celery/search_index_tasks.py | 30 ++++++++++++++++ src/scripts/algolia_search.js | 5 +++ 7 files changed, 98 insertions(+), 40 deletions(-) create mode 100644 pillar/celery/search_index_tasks.py diff --git a/pillar/__init__.py b/pillar/__init__.py index ba79fa48..646c5a83 100644 --- a/pillar/__init__.py +++ b/pillar/__init__.py @@ -466,7 +466,7 @@ class PillarServer(Eve): # Pillar-defined Celery task modules: celery_task_modules = [ 'pillar.celery.tasks', - 'pillar.celery.algolia_tasks', + 'pillar.celery.search_index_tasks', 'pillar.celery.file_link_tasks', ] diff --git a/pillar/api/nodes/__init__.py b/pillar/api/nodes/__init__.py index 27437ce6..1b00fa8b 100644 --- a/pillar/api/nodes/__init__.py +++ b/pillar/api/nodes/__init__.py @@ -167,7 +167,8 @@ def create_short_code(node) -> str: def short_link_info(short_code): """Returns the short link info in a dict.""" - short_link = urllib.parse.urljoin(current_app.config['SHORT_LINK_BASE_URL'], short_code) + short_link = urllib.parse.urljoin( + current_app.config['SHORT_LINK_BASE_URL'], short_code) return { 'short_code': short_code, @@ -185,7 +186,7 @@ def after_replacing_node(item, original): project is private, prevent public indexing. """ - from pillar.celery import algolia_tasks + from pillar.celery import search_index_tasks as index projects_collection = current_app.data.driver.db['projects'] project = projects_collection.find_one({'_id': item['project']}) @@ -195,10 +196,11 @@ def after_replacing_node(item, original): status = item['properties'].get('status', 'unpublished') node_id = str(item['_id']) + if status == 'published': - algolia_tasks.algolia_index_node_save.delay(node_id) + index.node_save.delay(node_id) else: - algolia_tasks.algolia_index_node_delete.delay(node_id) + index.node_delete.delay(node_id) def before_inserting_nodes(items): @@ -372,8 +374,8 @@ def before_deleting_node(node: dict): def after_deleting_node(item): - from pillar.celery import algolia_tasks - algolia_tasks.algolia_index_node_delete.delay(str(item['_id'])) + from pillar.celery import search_index_tasks as index + index.node_delete.delay(str(item['_id'])) only_for_comments = only_for_node_type_decorator('comment') diff --git a/pillar/api/users/hooks.py b/pillar/api/users/hooks.py index 46e4ef9a..7bdf75d4 100644 --- a/pillar/api/users/hooks.py +++ b/pillar/api/users/hooks.py @@ -61,19 +61,26 @@ def before_replacing_user(request, lookup): # Regular users should always have an email address if 'service' not in put_data.get('roles', ()): if not put_data.get('email'): - raise wz_exceptions.UnprocessableEntity('email field must be given') + raise wz_exceptions.UnprocessableEntity( + 'email field must be given') def push_updated_user_to_algolia(user, original): - """Push an update to the Algolia index when a user item is updated""" + """ + Push an update to the Algolia index when a user + item is updated + """ - from pillar.celery import algolia_tasks + from pillar.celery import search_index_tasks as index - algolia_tasks.push_updated_user_to_algolia.delay(str(user['_id'])) + index.updated_user.delay(str(user['_id'])) def send_blinker_signal_roles_changed(user, original): - """Sends a Blinker signal that the user roles were changed, so others can respond.""" + """ + Sends a Blinker signal that the user roles were + changed, so others can respond. + """ current_roles = set(user.get('roles', [])) original_roles = set(original.get('roles', [])) diff --git a/pillar/api/utils/algolia.py b/pillar/api/utils/algolia.py index 7ade679f..f5e8abcc 100644 --- a/pillar/api/utils/algolia.py +++ b/pillar/api/utils/algolia.py @@ -35,7 +35,31 @@ def algolia_index_user_save(user): 'email': user['email'] }) - log.debug('Pushed user %r to Algolia index %r', user['_id'], index_users.index_name) + log.debug( + 'Pushed user %r to Algolia index %r', + user['_id'], index_users.index_name) + + +def _handle_picture(node, doc): + """ + add picture fields to be indexed + """ + + if 'picture' in node and node['picture']: + files_collection = current_app.data.driver.db['files'] + lookup = {'_id': ObjectId(node['picture'])} + picture = files_collection.find_one(lookup) + + img_variation_t = next( + (item for item in picture['variations'] + if item['size'] == 't'), None) + + if img_variation_t: + doc['picture'] = generate_link( + picture['backend'], + img_variation_t['file_path'], + project_id=str(picture['project']), + is_public=True) @skip_when_testing @@ -54,7 +78,7 @@ def algolia_index_node_save(node): users_collection = current_app.data.driver.db['users'] user = users_collection.find_one({'_id': ObjectId(node['user'])}) - node_ob = { + doc = { 'objectID': node['_id'], 'name': node['name'], 'project': { @@ -69,35 +93,27 @@ def algolia_index_node_save(node): 'full_name': user['full_name'] }, } + if 'description' in node and node['description']: - node_ob['description'] = node['description'] - if 'picture' in node and node['picture']: - files_collection = current_app.data.driver.db['files'] - lookup = {'_id': ObjectId(node['picture'])} - picture = files_collection.find_one(lookup) - if picture['backend'] == 'gcs': - variation_t = next((item for item in picture['variations'] \ - if item['size'] == 't'), None) - if variation_t: - node_ob['picture'] = generate_link(picture['backend'], - variation_t['file_path'], - project_id=str(picture['project']), - is_public=True) + doc['description'] = node['description'] + + _handle_picture(node, doc) + # If the node has world permissions, compute the Free permission if 'permissions' in node and 'world' in node['permissions']: if 'GET' in node['permissions']['world']: - node_ob['is_free'] = True + doc['is_free'] = True # Append the media key if the node is of node_type 'asset' if node['node_type'] == 'asset': - node_ob['media'] = node['properties']['content_type'] + doc['media'] = node['properties']['content_type'] # Add extra properties for prop in ('tags', 'license_notes'): if prop in node['properties']: - node_ob[prop] = node['properties'][prop] + doc[prop] = node['properties'][prop] - current_app.algolia_index_nodes.save_object(node_ob) + current_app.algolia_index_nodes.save_object(doc) @skip_when_testing diff --git a/pillar/celery/algolia_tasks.py b/pillar/celery/algolia_tasks.py index 0bc61173..5218e8f0 100644 --- a/pillar/celery/algolia_tasks.py +++ b/pillar/celery/algolia_tasks.py @@ -8,7 +8,6 @@ from pillar import current_app log = logging.getLogger(__name__) -@current_app.celery.task(ignore_result=True) def push_updated_user_to_algolia(user_id: str): """Push an update to the Algolia index when a user item is updated""" @@ -25,12 +24,11 @@ def push_updated_user_to_algolia(user_id: str): try: algolia_index_user_save(user) except AlgoliaException as ex: - log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s', + log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s', # noqa user.get('username'), user_id, ex) -@current_app.celery.task(ignore_result=True) -def algolia_index_node_save(node_id: str): +def index_node_save(node_id: str): from pillar.api.utils.algolia import algolia_index_node_save node_oid = bson.ObjectId(node_id) @@ -46,17 +44,17 @@ def algolia_index_node_save(node_id: str): try: algolia_index_node_save(node) except AlgoliaException as ex: - log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex) + log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex) # noqa -@current_app.celery.task(ignore_result=True) -def algolia_index_node_delete(node_id: str): +def index_node_delete(node_id: str): from pillar.api.utils.algolia import algolia_index_node_delete - # Deleting a node takes nothing more than the ID anyway. No need to fetch anything from Mongo. + # Deleting a node takes nothing more than the ID anyway. + # No need to fetch anything from Mongo. fake_node = {'_id': bson.ObjectId(node_id)} try: algolia_index_node_delete(fake_node) except AlgoliaException as ex: - log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex) + log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex) # noqa diff --git a/pillar/celery/search_index_tasks.py b/pillar/celery/search_index_tasks.py new file mode 100644 index 00000000..769e331f --- /dev/null +++ b/pillar/celery/search_index_tasks.py @@ -0,0 +1,30 @@ +import logging + +from . import algolia_tasks + +from pillar import current_app + +log = logging.getLogger(__name__) + + +# TODO make index backend conditional on settings. +# now uses angolia, but should use elastic + + +@current_app.celery.task(ignore_result=True) +def updated_user(user_id: str): + """Push an update to the index when a user item is updated""" + + algolia_tasks.push_updated_user_to_algolia(user_id) + + +@current_app.celery.task(ignore_result=True) +def node_save(node_id: str): + + algolia_tasks.index_node_save(node_id) + + +@current_app.celery.task(ignore_result=True) +def node_delete(node_id: str): + + algolia_tasks.index_node_delete(node_id) diff --git a/src/scripts/algolia_search.js b/src/scripts/algolia_search.js index 9e1ff119..d1b445c6 100644 --- a/src/scripts/algolia_search.js +++ b/src/scripts/algolia_search.js @@ -21,6 +21,7 @@ $(document).ready(function() { var sliderTemplate = Hogan.compile($('#slider-template').text()); var paginationTemplate = Hogan.compile($('#pagination-template').text()); + // replace with something elasticy! // Client initialization var algolia = algoliasearch(APPLICATION_ID, SEARCH_ONLY_API_KEY); @@ -36,6 +37,7 @@ $(document).ready(function() { }) }; + // replace with something elastici! // Setup the search helper var helper = algoliasearchHelper(algolia, INDEX_NAME, params); @@ -43,6 +45,7 @@ $(document).ready(function() { var result = $.grep(FACET_CONFIG, function(e) { return e.hidden && e.hidden == true; }); + for (var i = 0; i < result.length; i++) { var f = result[i]; helper.addFacetRefinement(f.name, f.value); @@ -60,9 +63,11 @@ $(document).ready(function() { helper.on('change', function(state) { setURLParams(state); }); + helper.on('error', function(error) { console.log(error); }); + helper.on('result', function(content, state) { renderStats(content); renderHits(content);