From 741cdf6e129725ce15b00b7387f9c2f9e12fcdf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Fri, 26 Jan 2018 10:29:28 +0100 Subject: [PATCH] Elastic: regenerate picture URL before inserting into ElasticSearch This ensures the thumbnail URL is public so that it won't expire. Since this now requires API calls to Google, I've increased the number of parallel threads used for indexing, since they'll be waiting for network I/O more. --- pillar/api/file_storage/__init__.py | 2 +- pillar/celery/search_index_tasks.py | 16 ++++++++++------ pillar/cli/elastic.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pillar/api/file_storage/__init__.py b/pillar/api/file_storage/__init__.py index 30c8f40c..13457afe 100644 --- a/pillar/api/file_storage/__init__.py +++ b/pillar/api/file_storage/__init__.py @@ -338,7 +338,7 @@ def process_file(bucket: Bucket, file_id, status, r) -def generate_link(backend, file_path: str, project_id=None, is_public=False) -> str: +def generate_link(backend, file_path: str, project_id: str=None, is_public=False) -> str: """Hook to check the backend of a file resource, to build an appropriate link that can be used by the client to retrieve the actual file. """ diff --git a/pillar/celery/search_index_tasks.py b/pillar/celery/search_index_tasks.py index 09370b4c..d0d838df 100644 --- a/pillar/celery/search_index_tasks.py +++ b/pillar/celery/search_index_tasks.py @@ -29,21 +29,25 @@ def _get_node_from_id(node_id: str): def _handle_picture(node: dict, to_index: dict): - """ - add picture fields to be indexed - """ + """Add picture URL in-place to the to-be-indexed node.""" - if not node.get('picture'): + picture_id = node.get('picture') + if not picture_id: return files_collection = current_app.data.driver.db['files'] - lookup = {'_id': ObjectId(node['picture'])} + lookup = {'_id': ObjectId(picture_id)} picture = files_collection.find_one(lookup) for item in picture.get('variations', []): if item['size'] != 't': continue - to_index['picture'] = item['link'] + + link = generate_link(picture['backend'], + item['file_path'], + str(picture['project']), + is_public=True) + to_index['picture'] = link break diff --git a/pillar/cli/elastic.py b/pillar/cli/elastic.py index b7237b2c..106d99fa 100644 --- a/pillar/cli/elastic.py +++ b/pillar/cli/elastic.py @@ -17,7 +17,7 @@ name_to_task = { 'nodes': index.ResetNodeIndex, 'users': index.ResetUserIndex, } -REINDEX_THREAD_COUNT = 3 +REINDEX_THREAD_COUNT = 5 @manager_elastic.option('indices', nargs='*')