wip D2950

pillar/api/search/elastic_indexing.py

@@ -12,7 +12,7 @@ elk_hosts = current_app.config['ELASTIC_SEARCH_HOSTS']
 
 connections.create_connection(
     hosts=elk_hosts,
-    sniff_on_start=True,
+    sniff_on_start=False,
     timeout=20)
 
 

pillar/api/search/index.py

@@ -17,7 +17,8 @@ class ResetIndexTask(object):
     index_key = ''
     """Key into the ELASTIC_INDICES dict in the app config."""
 
-    doc_types: List[type]  = []
+    # List of elastic document  types
+    doc_types = []
     name = 'remove index'
 
     def __init__(self):

pillar/cli/__init__.py

@@ -12,7 +12,7 @@ from pillar.cli.celery import manager_celery
 from pillar.cli.maintenance import manager_maintenance
 from pillar.cli.operations import manager_operations
 from pillar.cli.setup import manager_setup
-from pillar.cli.elastic import manager_elk
+from pillar.cli.elastic import manager_elastic
 
 from pillar.cli import translations
 
@@ -23,4 +23,4 @@ manager.add_command('celery', manager_celery)
 manager.add_command("maintenance", manager_maintenance)
 manager.add_command("setup", manager_setup)
 manager.add_command("operations", manager_operations)
-manager.add_command("elastic", manager_elk)
+manager.add_command("elastic", manager_elastic)

pillar/cli/elastic.py

@@ -9,8 +9,8 @@ from pillar.api.search import index
 
 log = logging.getLogger(__name__)
 
-manager_elk = Manager(
-    current_app, usage="Elastic utilities, like reset_index()")
+manager_elastic = Manager(
+    current_app, usage="Elastic utilities")
 
 name_to_task = {
     'nodes': index.ResetNodeIndex,
@@ -18,12 +18,12 @@ name_to_task = {
 }
 
 
-@manager_elk.option('indices', nargs='*')
+@manager_elastic.option('indices', nargs='*')
 def reset_index(indices):
     """
     Destroy and recreate elastic indices
 
-    node, user ...
+    nodes, users
     """
 
     with current_app.app_context():
@@ -44,7 +44,7 @@ def _reindex_users():
     users_coll = db['users']
     user_count = users_coll.count()
 
-    log.debug('Reindexing %d in Elastic', user_count)
+    log.debug('Reindexing users %d in Elastic', user_count)
 
     from pillar.celery.search_index_tasks import prepare_user_data
     from pillar.api.search import elastic_indexing
@@ -62,7 +62,6 @@ def _reindex_users():
             continue
 
 
-# stolen from api.latest.
 def _public_project_ids() -> typing.List[bson.ObjectId]:
     """Returns a list of ObjectIDs of public projects.
 
@@ -76,15 +75,11 @@ def _public_project_ids() -> typing.List[bson.ObjectId]:
 
 def _reindex_nodes():
     db = current_app.db()
-    pipeline = [
-        {'$match': {'project': {'$in': _public_project_ids()}}},
-    ]
-    private_filter = {'project': {'$in': _public_project_ids()}}
     nodes_coll = db['nodes']
-    nodes_coll = nodes_coll.find(private_filter)
+    nodes_coll = nodes_coll.find({'project': {'$in': _public_project_ids()}})
     node_count = nodes_coll.count()
 
-    log.debug('Reindexing %d in Elastic', node_count)
+    log.debug('Reindexing nodes %d in Elastic', node_count)
 
     from pillar.celery.search_index_tasks import prepare_node_data
     from pillar.api.search import elastic_indexing
@@ -93,12 +88,12 @@ def _reindex_nodes():
         try:
             to_index = prepare_node_data('', node=node)
             elastic_indexing.index_node_save(to_index)
-        except(KeyError, AttributeError):
+        except (KeyError, AttributeError):
             log.exception('Field is missing for %s', node)
             continue
 
 
-@manager_elk.option('indexname', nargs='?')
+@manager_elastic.option('indexname', nargs='?')
 def reindex(indexname=''):
     if not indexname:
         log.info('reindex everything..')

pillar/config.py

@@ -78,10 +78,11 @@ ALGOLIA_API_KEY = '-SECRET-'
 ALGOLIA_INDEX_USERS = 'dev_Users'
 ALGOLIA_INDEX_NODES = 'dev_Nodes'
 
-SEARCH_BACKENDS = ['algolia', 'elastic']
+SEARCH_BACKENDS = ('algolia', 'elastic')    # search backend we use
 
-ELASTIC_SEARCH_HOSTS = ['elasticsearch']
+ELASTIC_SEARCH_HOSTS = ['elasticsearch']    # elasticsearch hosts
 ELASTIC_INDICES = {
+    # elasticsearch indexes
     'NODE': 'nodes',
     'USER': 'users',
 }