Migrated Algolia push/delete of nodes to Celery background tasks.

pillar/api/nodes/__init__.py

@@ -13,8 +13,6 @@ from pillar.api.activities import activity_subscribe, activity_object_add
 from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
 from pillar.api.file_storage_backends.gcs import update_file_name
 from pillar.api.utils import str2id, jsonify
-from pillar.api.utils.algolia import algolia_index_node_delete
-from pillar.api.utils.algolia import algolia_index_node_save
 from pillar.api.utils.authorization import check_permissions, require_login
 
 log = logging.getLogger(__name__)
@@ -187,27 +185,20 @@ def after_replacing_node(item, original):
     project is private, prevent public indexing.
     """
 
+    from pillar.celery import algolia_tasks
+
     projects_collection = current_app.data.driver.db['projects']
     project = projects_collection.find_one({'_id': item['project']})
     if project.get('is_private', False):
         # Skip index updating and return
         return
 
-    from algoliasearch.helpers import AlgoliaException
     status = item['properties'].get('status', 'unpublished')
-
+    node_id = str(item['_id'])
     if status == 'published':
-        try:
-            algolia_index_node_save(item)
-        except AlgoliaException as ex:
-            log.warning('Unable to push node info to Algolia for node %s; %s',
-                        item.get('_id'), ex)
+        algolia_tasks.algolia_index_node_save.delay(node_id)
     else:
-        try:
-            algolia_index_node_delete(item)
-        except AlgoliaException as ex:
-            log.warning('Unable to delete node info to Algolia for node %s; %s',
-                        item.get('_id'), ex)
+        algolia_tasks.algolia_index_node_delete.delay(node_id)
 
 
 def before_inserting_nodes(items):
@@ -375,12 +366,8 @@ def nodes_set_default_picture(nodes):
 
 
 def after_deleting_node(item):
-    from algoliasearch.helpers import AlgoliaException
-    try:
-        algolia_index_node_delete(item)
-    except AlgoliaException as ex:
-        log.warning('Unable to delete node info to Algolia for node %s; %s',
-                    item.get('_id'), ex)
+    from pillar.celery import algolia_tasks
+    algolia_tasks.algolia_index_node_delete.delay(str(item['_id']))
 
 
 only_for_comments = only_for_node_type_decorator('comment')

pillar/celery/algolia_tasks.py

@@ -1,5 +1,6 @@
 import logging
 
+from algoliasearch.helpers import AlgoliaException
 import bson
 
 from pillar import current_app
@@ -11,16 +12,51 @@ log = logging.getLogger(__name__)
 def push_updated_user_to_algolia(user_id: str):
     """Push an update to the Algolia index when a user item is updated"""
 
-    from algoliasearch.helpers import AlgoliaException
     from pillar.api.utils.algolia import algolia_index_user_save
 
     user_oid = bson.ObjectId(user_id)
     log.info('Retrieving user %s', user_oid)
     users_coll = current_app.db('users')
     user = users_coll.find_one({'_id': user_oid})
+    if user is None:
+        log.warning('Unable to find user %s, not updating Algolia.', user_oid)
+        return
 
     try:
         algolia_index_user_save(user)
     except AlgoliaException as ex:
         log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',
-                    user.get('username'), user.get('_id'), ex)
+                    user.get('username'), user_id, ex)
+
+
+@current_app.celery.task(ignore_result=True)
+def algolia_index_node_save(node_id: str):
+    from pillar.api.utils.algolia import algolia_index_node_save
+
+    node_oid = bson.ObjectId(node_id)
+    log.info('Retrieving node %s', node_oid)
+
+    nodes_coll = current_app.db('nodes')
+    node = nodes_coll.find_one({'_id': node_oid})
+
+    if node is None:
+        log.warning('Unable to find node %s, not updating Algolia.', node_id)
+        return
+
+    try:
+        algolia_index_node_save(node)
+    except AlgoliaException as ex:
+        log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex)
+
+
+@current_app.celery.task(ignore_result=True)
+def algolia_index_node_delete(node_id: str):
+    from pillar.api.utils.algolia import algolia_index_node_delete
+
+    # Deleting a node takes nothing more than the ID anyway. No need to fetch anything from Mongo.
+    fake_node = {'_id': bson.ObjectId(node_id)}
+
+    try:
+        algolia_index_node_delete(fake_node)
+    except AlgoliaException as ex:
+        log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex)