archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Migrated Algolia push/delete of nodes to Celery background tasks.

Browse Source
This commit is contained in:
Sybren A. Stüvel 2017-06-01 16:39:59 +02:00
parent d0c30cfeca
commit 878bf22695
2 changed files with 45 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pillar/api/nodes/__init__.py
View File

@ -13,8 +13,6 @@ from pillar.api.activities import activity_subscribe, activity_object_add
from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES from pillar.api.node_types import PILLAR_NAMED_NODE_TYPES
from pillar.api.file_storage_backends.gcs import update_file_name from pillar.api.file_storage_backends.gcs import update_file_name
from pillar.api.utils import str2id, jsonify from pillar.api.utils import str2id, jsonify
from pillar.api.utils.algolia import algolia_index_node_delete
from pillar.api.utils.algolia import algolia_index_node_save
from pillar.api.utils.authorization import check_permissions, require_login from pillar.api.utils.authorization import check_permissions, require_login
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -187,27 +185,20 @@ def after_replacing_node(item, original):
project is private, prevent public indexing. project is private, prevent public indexing.
""" """
from pillar.celery import algolia_tasks
projects_collection = current_app.data.driver.db['projects'] projects_collection = current_app.data.driver.db['projects']
project = projects_collection.find_one({'_id': item['project']}) project = projects_collection.find_one({'_id': item['project']})
if project.get('is_private', False): if project.get('is_private', False):
# Skip index updating and return # Skip index updating and return
return return
from algoliasearch.helpers import AlgoliaException
status = item['properties'].get('status', 'unpublished') status = item['properties'].get('status', 'unpublished')
node_id = str(item['_id'])
if status == 'published': if status == 'published':
try: algolia_tasks.algolia_index_node_save.delay(node_id)
algolia_index_node_save(item)
except AlgoliaException as ex:
log.warning('Unable to push node info to Algolia for node %s; %s',
item.get('_id'), ex)
else: else:
try: algolia_tasks.algolia_index_node_delete.delay(node_id)
algolia_index_node_delete(item)
except AlgoliaException as ex:
log.warning('Unable to delete node info to Algolia for node %s; %s',
item.get('_id'), ex)
def before_inserting_nodes(items): def before_inserting_nodes(items):
@ -375,12 +366,8 @@ def nodes_set_default_picture(nodes):
def after_deleting_node(item): def after_deleting_node(item):
from algoliasearch.helpers import AlgoliaException from pillar.celery import algolia_tasks
try: algolia_tasks.algolia_index_node_delete.delay(str(item['_id']))
algolia_index_node_delete(item)
except AlgoliaException as ex:
log.warning('Unable to delete node info to Algolia for node %s; %s',
item.get('_id'), ex)
only_for_comments = only_for_node_type_decorator('comment') only_for_comments = only_for_node_type_decorator('comment')

40
pillar/celery/algolia_tasks.py
View File

@ -1,5 +1,6 @@
import logging import logging
from algoliasearch.helpers import AlgoliaException
import bson import bson
from pillar import current_app from pillar import current_app
@ -11,16 +12,51 @@ log = logging.getLogger(__name__)
def push_updated_user_to_algolia(user_id: str): def push_updated_user_to_algolia(user_id: str):
"""Push an update to the Algolia index when a user item is updated""" """Push an update to the Algolia index when a user item is updated"""
from algoliasearch.helpers import AlgoliaException
from pillar.api.utils.algolia import algolia_index_user_save from pillar.api.utils.algolia import algolia_index_user_save
user_oid = bson.ObjectId(user_id) user_oid = bson.ObjectId(user_id)
log.info('Retrieving user %s', user_oid) log.info('Retrieving user %s', user_oid)
users_coll = current_app.db('users') users_coll = current_app.db('users')
user = users_coll.find_one({'_id': user_oid}) user = users_coll.find_one({'_id': user_oid})
if user is None:
log.warning('Unable to find user %s, not updating Algolia.', user_oid)
return
try: try:
algolia_index_user_save(user) algolia_index_user_save(user)
except AlgoliaException as ex: except AlgoliaException as ex:
log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s', log.warning('Unable to push user info to Algolia for user "%s", id=%s; %s',
user.get('username'), user.get('_id'), ex) user.get('username'), user_id, ex)
@current_app.celery.task(ignore_result=True)
def algolia_index_node_save(node_id: str):
from pillar.api.utils.algolia import algolia_index_node_save
node_oid = bson.ObjectId(node_id)
log.info('Retrieving node %s', node_oid)
nodes_coll = current_app.db('nodes')
node = nodes_coll.find_one({'_id': node_oid})
if node is None:
log.warning('Unable to find node %s, not updating Algolia.', node_id)
return
try:
algolia_index_node_save(node)
except AlgoliaException as ex:
log.warning('Unable to push node info to Algolia for node %s; %s', node_id, ex)
@current_app.celery.task(ignore_result=True)
def algolia_index_node_delete(node_id: str):
from pillar.api.utils.algolia import algolia_index_node_delete
# Deleting a node takes nothing more than the ID anyway. No need to fetch anything from Mongo.
fake_node = {'_id': bson.ObjectId(node_id)}
try:
algolia_index_node_delete(fake_node)
except AlgoliaException as ex:
log.warning('Unable to delete node info to Algolia for node %s; %s', node_id, ex)