T53161 start working on elastic..

pillar/__init__.py

@@ -466,7 +466,7 @@ class PillarServer(Eve):
         # Pillar-defined Celery task modules:
         celery_task_modules = [
             'pillar.celery.tasks',
-            'pillar.celery.algolia_tasks',
+            'pillar.celery.search_index_tasks',
             'pillar.celery.file_link_tasks',
         ]
 

pillar/api/nodes/__init__.py

@@ -167,7 +167,8 @@ def create_short_code(node) -> str:
 def short_link_info(short_code):
     """Returns the short link info in a dict."""
 
-    short_link = urllib.parse.urljoin(current_app.config['SHORT_LINK_BASE_URL'], short_code)
+    short_link = urllib.parse.urljoin(
+        current_app.config['SHORT_LINK_BASE_URL'], short_code)
 
     return {
         'short_code': short_code,
@@ -185,7 +186,7 @@ def after_replacing_node(item, original):
     project is private, prevent public indexing.
     """
 
-    from pillar.celery import algolia_tasks
+    from pillar.celery import search_index_tasks as index
 
     projects_collection = current_app.data.driver.db['projects']
     project = projects_collection.find_one({'_id': item['project']})
@@ -195,10 +196,11 @@ def after_replacing_node(item, original):
 
     status = item['properties'].get('status', 'unpublished')
     node_id = str(item['_id'])
+
     if status == 'published':
-        algolia_tasks.algolia_index_node_save.delay(node_id)
+        index.node_save.delay(node_id)
     else:
-        algolia_tasks.algolia_index_node_delete.delay(node_id)
+        index.node_delete.delay(node_id)
 
 
 def before_inserting_nodes(items):
@@ -372,8 +374,8 @@ def before_deleting_node(node: dict):
 
 
 def after_deleting_node(item):
-    from pillar.celery import algolia_tasks
+    from pillar.celery import search_index_tasks as index
-    algolia_tasks.algolia_index_node_delete.delay(str(item['_id']))
+    index.node_delete.delay(str(item['_id']))
 
 
 only_for_comments = only_for_node_type_decorator('comment')

pillar/api/users/hooks.py

@@ -61,19 +61,26 @@ def before_replacing_user(request, lookup):
     # Regular users should always have an email address
     if 'service' not in put_data.get('roles', ()):
         if not put_data.get('email'):
-            raise wz_exceptions.UnprocessableEntity('email field must be given')
+            raise wz_exceptions.UnprocessableEntity(
+                'email field must be given')
 
 
 def push_updated_user_to_algolia(user, original):
-    """Push an update to the Algolia index when a user item is updated"""
+    """
+    Push an update to the Algolia index when a user
+    item is updated
+    """
 
-    from pillar.celery import algolia_tasks
+    from pillar.celery import search_index_tasks as index
 
-    algolia_tasks.push_updated_user_to_algolia.delay(str(user['_id']))
+    index.updated_user.delay(str(user['_id']))
 
 
 def send_blinker_signal_roles_changed(user, original):
-    """Sends a Blinker signal that the user roles were changed, so others can respond."""
+    """
+    Sends a Blinker signal that the user roles were
+    changed, so others can respond.
+    """
 
     current_roles = set(user.get('roles', []))
     original_roles = set(original.get('roles', []))

pillar/api/utils/algolia.py

@@ -35,7 +35,31 @@ def algolia_index_user_save(user):
         'email': user['email']
     })
 
-    log.debug('Pushed user %r to Algolia index %r', user['_id'], index_users.index_name)
+    log.debug(
+        'Pushed user %r to Algolia index %r',
+        user['_id'], index_users.index_name)
+
+
+def _handle_picture(node, doc):
+    """
+    add picture fields to be indexed
+    """
+
+    if 'picture' in node and node['picture']:
+        files_collection = current_app.data.driver.db['files']
+        lookup = {'_id': ObjectId(node['picture'])}
+        picture = files_collection.find_one(lookup)
+
+        img_variation_t = next(
+            (item for item in picture['variations']
+             if item['size'] == 't'), None)
+
+        if img_variation_t:
+            doc['picture'] = generate_link(
+                picture['backend'],
+                img_variation_t['file_path'],
+                project_id=str(picture['project']),
+                is_public=True)
 
 
 @skip_when_testing
@@ -54,7 +78,7 @@ def algolia_index_node_save(node):
     users_collection = current_app.data.driver.db['users']
     user = users_collection.find_one({'_id': ObjectId(node['user'])})
 
-    node_ob = {
+    doc = {
         'objectID': node['_id'],
         'name': node['name'],
         'project': {
@@ -69,35 +93,27 @@ def algolia_index_node_save(node):
             'full_name': user['full_name']
         },
     }
+
     if 'description' in node and node['description']:
-        node_ob['description'] = node['description']
+        doc['description'] = node['description']
-    if 'picture' in node and node['picture']:
+
-        files_collection = current_app.data.driver.db['files']
+    _handle_picture(node, doc)
-        lookup = {'_id': ObjectId(node['picture'])}
+
-        picture = files_collection.find_one(lookup)
-        if picture['backend'] == 'gcs':
-            variation_t = next((item for item in picture['variations'] \
-                                if item['size'] == 't'), None)
-            if variation_t:
-                node_ob['picture'] = generate_link(picture['backend'],
-                                                   variation_t['file_path'],
-                                                   project_id=str(picture['project']),
-                                                   is_public=True)
     # If the node has world permissions, compute the Free permission
     if 'permissions' in node and 'world' in node['permissions']:
         if 'GET' in node['permissions']['world']:
-            node_ob['is_free'] = True
+            doc['is_free'] = True
 
     # Append the media key if the node is of node_type 'asset'
     if node['node_type'] == 'asset':
-        node_ob['media'] = node['properties']['content_type']
+        doc['media'] = node['properties']['content_type']
 
     # Add extra properties
     for prop in ('tags', 'license_notes'):
         if prop in node['properties']:
-            node_ob[prop] = node['properties'][prop]
+            doc[prop] = node['properties'][prop]
 
-    current_app.algolia_index_nodes.save_object(node_ob)
+    current_app.algolia_index_nodes.save_object(doc)
 
 
 @skip_when_testing

pillar/celery/algolia_tasks.py

@@ -8,7 +8,6 @@ from pillar import current_app
 log = logging.getLogger(__name__)
 
 
-@current_app.celery.task(ignore_result=True)
 def push_updated_user_to_algolia(user_id: str):
     """Push an update to the Algolia index when a user item is updated"""
 
@@ -25,12 +24,11 @@ def push_updated_user_to_algolia(user_id: str):
     try:
         algolia_index_user_save(user)
     except AlgoliaException as ex:
-        log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',
+        log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',  # noqa
                     user.get('username'), user_id, ex)
 
 
-@current_app.celery.task(ignore_result=True)
+def index_node_save(node_id: str):
-def algolia_index_node_save(node_id: str):
     from pillar.api.utils.algolia import algolia_index_node_save
 
     node_oid = bson.ObjectId(node_id)
@@ -46,17 +44,17 @@ def algolia_index_node_save(node_id: str):
     try:
         algolia_index_node_save(node)
     except AlgoliaException as ex:
-        log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex)
+        log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex)  # noqa
 
 
-@current_app.celery.task(ignore_result=True)
+def index_node_delete(node_id: str):
-def algolia_index_node_delete(node_id: str):
     from pillar.api.utils.algolia import algolia_index_node_delete
 
-    # Deleting a node takes nothing more than the ID anyway. No need to fetch anything from Mongo.
+    # Deleting a node takes nothing more than the ID anyway.
+    # No need to fetch anything from Mongo.
     fake_node = {'_id': bson.ObjectId(node_id)}
 
     try:
         algolia_index_node_delete(fake_node)
     except AlgoliaException as ex:
-        log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex)
+        log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex)  # noqa

pillar/celery/search_index_tasks.py

@@ -0,0 +1,30 @@
+import logging
+
+from . import algolia_tasks
+
+from pillar import current_app
+
+log = logging.getLogger(__name__)
+
+
+# TODO make index backend conditional on settings.
+# now uses angolia, but should use elastic
+
+
+@current_app.celery.task(ignore_result=True)
+def updated_user(user_id: str):
+    """Push an update to the index when a user item is updated"""
+
+    algolia_tasks.push_updated_user_to_algolia(user_id)
+
+
+@current_app.celery.task(ignore_result=True)
+def node_save(node_id: str):
+
+    algolia_tasks.index_node_save(node_id)
+
+
+@current_app.celery.task(ignore_result=True)
+def node_delete(node_id: str):
+
+    algolia_tasks.index_node_delete(node_id)

src/scripts/algolia_search.js

@@ -21,6 +21,7 @@ $(document).ready(function() {
     var sliderTemplate = Hogan.compile($('#slider-template').text());
     var paginationTemplate = Hogan.compile($('#pagination-template').text());
 
+    // replace with something elasticy!
     // Client initialization
     var algolia = algoliasearch(APPLICATION_ID, SEARCH_ONLY_API_KEY);
 
@@ -36,6 +37,7 @@ $(document).ready(function() {
         })
     };
 
+    // replace with something elastici!
     // Setup the search helper
     var helper = algoliasearchHelper(algolia, INDEX_NAME, params);
 
@@ -43,6 +45,7 @@ $(document).ready(function() {
     var result = $.grep(FACET_CONFIG, function(e) {
         return e.hidden && e.hidden == true;
     });
+
     for (var i = 0; i < result.length; i++) {
         var f = result[i];
         helper.addFacetRefinement(f.name, f.value);
@@ -60,9 +63,11 @@ $(document).ready(function() {
     helper.on('change', function(state) {
         setURLParams(state);
     });
+
     helper.on('error', function(error) {
         console.log(error);
     });
+
     helper.on('result', function(content, state) {
         renderStats(content);
         renderHits(content);