Elastic: regenerate picture URL before inserting into ElasticSearch

This ensures the thumbnail URL is public so that it won't expire.
Since this now requires API calls to Google, I've increased the number of
parallel threads used for indexing, since they'll be waiting for network
I/O more.

pillar/api/file_storage/__init__.py

@@ -338,7 +338,7 @@ def process_file(bucket: Bucket,
                     file_id, status, r)
 
 
-def generate_link(backend, file_path: str, project_id=None, is_public=False) -> str:
+def generate_link(backend, file_path: str, project_id: str=None, is_public=False) -> str:
     """Hook to check the backend of a file resource, to build an appropriate link
     that can be used by the client to retrieve the actual file.
     """

pillar/celery/search_index_tasks.py

@@ -29,21 +29,25 @@ def _get_node_from_id(node_id: str):
 
 
 def _handle_picture(node: dict, to_index: dict):
-    """
-    add picture fields to be indexed
-    """
+    """Add picture URL in-place to the to-be-indexed node."""
 
-    if not node.get('picture'):
+    picture_id = node.get('picture')
+    if not picture_id:
         return
 
     files_collection = current_app.data.driver.db['files']
-    lookup = {'_id': ObjectId(node['picture'])}
+    lookup = {'_id': ObjectId(picture_id)}
     picture = files_collection.find_one(lookup)
 
     for item in picture.get('variations', []):
         if item['size'] != 't':
             continue
-        to_index['picture'] = item['link']
+
+        link = generate_link(picture['backend'],
+                             item['file_path'],
+                             str(picture['project']),
+                             is_public=True)
+        to_index['picture'] = link
         break
 
 

pillar/cli/elastic.py

@@ -17,7 +17,7 @@ name_to_task = {
     'nodes': index.ResetNodeIndex,
     'users': index.ResetUserIndex,
 }
-REINDEX_THREAD_COUNT = 3
+REINDEX_THREAD_COUNT = 5
 
 
 @manager_elastic.option('indices', nargs='*')