T53161 proces feedback sybren, replace angolia with search

pillar/api/users/__init__.py

@@ -31,7 +31,7 @@ def add_user_to_group(user_id: bson.ObjectId, group_id: bson.ObjectId):
 
 def user_group_action(user_id: bson.ObjectId, group_id: bson.ObjectId, action: str):
     """Performs a group action (add/remove).
-    
+
     :param user_id: the user's ObjectID.
     :param group_id: the group's ObjectID.
     :param action: either '$pull' to remove from a group, or '$addToSet' to add to a group.
@@ -54,9 +54,9 @@ def user_group_action(user_id: bson.ObjectId, group_id: bson.ObjectId, action: s
                          f'user not found.')
 
 
-def _update_algolia_user_changed_role(sender, user: dict):
+def _update_search_user_changed_role(sender, user: dict):
     log.debug('Sending updated user %s to Algolia due to role change', user['_id'])
-    hooks.push_updated_user_to_algolia(user, original=None)
+    hooks.push_updated_user_to_search(user, original=None)
 
 
 def setup_app(app, api_prefix):
@@ -66,7 +66,7 @@ def setup_app(app, api_prefix):
     app.on_post_GET_users += hooks.post_GET_user
     app.on_pre_PUT_users += hooks.check_put_access
     app.on_pre_PUT_users += hooks.before_replacing_user
-    app.on_replaced_users += hooks.push_updated_user_to_algolia
+    app.on_replaced_users += hooks.push_updated_user_to_search
     app.on_replaced_users += hooks.send_blinker_signal_roles_changed
     app.on_fetched_item_users += hooks.after_fetching_user
     app.on_fetched_resource_users += hooks.after_fetching_user_resource
@@ -76,4 +76,4 @@ def setup_app(app, api_prefix):
 
     app.register_api_blueprint(blueprint_api, url_prefix=api_prefix)
 
-    service.signal_user_changed_role.connect(_update_algolia_user_changed_role)
+    service.signal_user_changed_role.connect(_update_search_user_changed_role)

pillar/api/users/hooks.py

@@ -65,15 +65,15 @@ def before_replacing_user(request, lookup):
                 'email field must be given')
 
 
-def push_updated_user_to_algolia(user, original):
+def push_updated_user_to_search(user, original):
     """
-    Push an update to the Algolia index when a user
+    Push an update to the Search index when a user
     item is updated
     """
 
-    from pillar.celery import search_index_tasks as index
+    from pillar.celery import search_index_tasks as searchindex
 
-    index.updated_user.delay(str(user['_id']))
+    searchindex.updated_user.delay(str(user['_id']))
 
 
 def send_blinker_signal_roles_changed(user, original):

pillar/api/utils/algolia.py

@@ -40,26 +40,28 @@ def algolia_index_user_save(user):
         user['_id'], index_users.index_name)
 
 
-def _handle_picture(node, doc):
+def _handle_picture(node: dict, to_index: dict):
     """
     add picture fields to be indexed
     """
 
-    if 'picture' in node and node['picture']:
-        files_collection = current_app.data.driver.db['files']
-        lookup = {'_id': ObjectId(node['picture'])}
-        picture = files_collection.find_one(lookup)
+    if not node.get('picture'):
+        return
 
-        img_variation_t = next(
-            (item for item in picture['variations']
-             if item['size'] == 't'), None)
+    files_collection = current_app.data.driver.db['files']
+    lookup = {'_id': ObjectId(node['picture'])}
+    picture = files_collection.find_one(lookup)
 
-        if img_variation_t:
-            doc['picture'] = generate_link(
-                picture['backend'],
-                img_variation_t['file_path'],
-                project_id=str(picture['project']),
-                is_public=True)
+    img_variation_t = next(
+        (item for item in picture['variations']
+         if item['size'] == 't'), None)
+
+    if img_variation_t:
+        to_index['picture'] = generate_link(
+            picture['backend'],
+            img_variation_t['file_path'],
+            project_id=str(picture['project']),
+            is_public=True)
 
 
 @skip_when_testing
@@ -78,7 +80,7 @@ def algolia_index_node_save(node):
     users_collection = current_app.data.driver.db['users']
     user = users_collection.find_one({'_id': ObjectId(node['user'])})
 
-    doc = {
+    to_index = {
         'objectID': node['_id'],
         'name': node['name'],
         'project': {
@@ -95,25 +97,25 @@ def algolia_index_node_save(node):
     }
 
     if 'description' in node and node['description']:
-        doc['description'] = node['description']
+        to_index['description'] = node['description']
 
-    _handle_picture(node, doc)
+    _handle_picture(node, to_index)
 
     # If the node has world permissions, compute the Free permission
-    if 'permissions' in node and 'world' in node['permissions']:
+    if 'world' in node.get('permissions', {}):
         if 'GET' in node['permissions']['world']:
-            doc['is_free'] = True
+            to_index['is_free'] = True
 
     # Append the media key if the node is of node_type 'asset'
     if node['node_type'] == 'asset':
-        doc['media'] = node['properties']['content_type']
+        to_index['media'] = node['properties']['content_type']
 
     # Add extra properties
     for prop in ('tags', 'license_notes'):
         if prop in node['properties']:
-            doc[prop] = node['properties'][prop]
+            to_index[prop] = node['properties'][prop]
 
-    current_app.algolia_index_nodes.save_object(doc)
+    current_app.algolia_index_nodes.save_object(to_index)
 
 
 @skip_when_testing

pillar/celery/algolia_indexing.py

@@ -8,7 +8,7 @@ from pillar import current_app
 log = logging.getLogger(__name__)
 
 
-def push_updated_user_to_algolia(user_id: str):
+def push_updated_user(user_id: str):
     """Push an update to the Algolia index when a user item is updated"""
 
     from pillar.api.utils.algolia import algolia_index_user_save
@@ -48,6 +48,7 @@ def index_node_save(node_id: str):
 
 
 def index_node_delete(node_id: str):
+
     from pillar.api.utils.algolia import algolia_index_node_delete
 
     # Deleting a node takes nothing more than the ID anyway.

pillar/celery/search_index_tasks.py

@@ -1,30 +1,45 @@
 import logging
-
-from . import algolia_tasks
-
 from pillar import current_app
 
+from . import algolia_indexing
+# from . import elastic_indexing
+
+
 log = logging.getLogger(__name__)
 
+# TODO(stephan) make index backend conditional on settings.
 
-# TODO make index backend conditional on settings.
-# now uses angolia, but should use elastic
+SEARCH_BACKENDS = {
+    'algolia': algolia_indexing,
+    'elastic': None,  # elastic_indexing
+}
 
 
 @current_app.celery.task(ignore_result=True)
 def updated_user(user_id: str):
     """Push an update to the index when a user item is updated"""
 
-    algolia_tasks.push_updated_user_to_algolia(user_id)
+    algolia_indexing.push_updated_user(user_id)
 
 
 @current_app.celery.task(ignore_result=True)
 def node_save(node_id: str):
 
-    algolia_tasks.index_node_save(node_id)
+    algolia_indexing.index_node_save(node_id)
 
 
 @current_app.celery.task(ignore_result=True)
 def node_delete(node_id: str):
 
-    algolia_tasks.index_node_delete(node_id)
+    algolia_indexing.index_node_delete(node_id)
+
+
+
+def build_doc_to_index_from(node: dict):
+    """
+    Given node build an to_index document
+    """
+    pass
+
+
+

pillar/config.py

@@ -75,6 +75,7 @@ ALGOLIA_INDEX_NODES = 'dev_Nodes'
 
 SEARCH_BACKEND = 'algolia'  # algolia, elastic
 
+
 ZENCODER_API_KEY = '-SECRET-'
 ZENCODER_NOTIFICATIONS_SECRET = '-SECRET-'
 ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'

src/scripts/algolia_search.js

@@ -2,6 +2,8 @@ $(document).ready(function() {
 
     /********************
      * INITIALIZATION
+     *
+     * TODO (stephan)
      * *******************/
 
     var HITS_PER_PAGE = 25;