diff --git a/pillar/api/search/elastic_indexing.py b/pillar/api/search/elastic_indexing.py index db8f88cf..c07e0ff8 100644 --- a/pillar/api/search/elastic_indexing.py +++ b/pillar/api/search/elastic_indexing.py @@ -12,7 +12,7 @@ elk_hosts = current_app.config['ELASTIC_SEARCH_HOSTS'] connections.create_connection( hosts=elk_hosts, - sniff_on_start=True, + sniff_on_start=False, timeout=20) diff --git a/pillar/api/search/index.py b/pillar/api/search/index.py index d05645c8..2efff9e1 100644 --- a/pillar/api/search/index.py +++ b/pillar/api/search/index.py @@ -17,7 +17,8 @@ class ResetIndexTask(object): index_key = '' """Key into the ELASTIC_INDICES dict in the app config.""" - doc_types: List[type] = [] + # List of elastic document types + doc_types = [] name = 'remove index' def __init__(self): diff --git a/pillar/cli/__init__.py b/pillar/cli/__init__.py index 1fec040d..421ed0e9 100644 --- a/pillar/cli/__init__.py +++ b/pillar/cli/__init__.py @@ -12,7 +12,7 @@ from pillar.cli.celery import manager_celery from pillar.cli.maintenance import manager_maintenance from pillar.cli.operations import manager_operations from pillar.cli.setup import manager_setup -from pillar.cli.elastic import manager_elk +from pillar.cli.elastic import manager_elastic from pillar.cli import translations @@ -23,4 +23,4 @@ manager.add_command('celery', manager_celery) manager.add_command("maintenance", manager_maintenance) manager.add_command("setup", manager_setup) manager.add_command("operations", manager_operations) -manager.add_command("elastic", manager_elk) +manager.add_command("elastic", manager_elastic) diff --git a/pillar/cli/elastic.py b/pillar/cli/elastic.py index c9c2d3f1..48e4ea6d 100644 --- a/pillar/cli/elastic.py +++ b/pillar/cli/elastic.py @@ -9,8 +9,8 @@ from pillar.api.search import index log = logging.getLogger(__name__) -manager_elk = Manager( - current_app, usage="Elastic utilities, like reset_index()") +manager_elastic = Manager( + current_app, usage="Elastic utilities") name_to_task = { 'nodes': index.ResetNodeIndex, @@ -18,12 +18,12 @@ name_to_task = { } -@manager_elk.option('indices', nargs='*') +@manager_elastic.option('indices', nargs='*') def reset_index(indices): """ Destroy and recreate elastic indices - node, user ... + nodes, users """ with current_app.app_context(): @@ -44,7 +44,7 @@ def _reindex_users(): users_coll = db['users'] user_count = users_coll.count() - log.debug('Reindexing %d in Elastic', user_count) + log.debug('Reindexing users %d in Elastic', user_count) from pillar.celery.search_index_tasks import prepare_user_data from pillar.api.search import elastic_indexing @@ -62,7 +62,6 @@ def _reindex_users(): continue -# stolen from api.latest. def _public_project_ids() -> typing.List[bson.ObjectId]: """Returns a list of ObjectIDs of public projects. @@ -76,15 +75,11 @@ def _public_project_ids() -> typing.List[bson.ObjectId]: def _reindex_nodes(): db = current_app.db() - pipeline = [ - {'$match': {'project': {'$in': _public_project_ids()}}}, - ] - private_filter = {'project': {'$in': _public_project_ids()}} nodes_coll = db['nodes'] - nodes_coll = nodes_coll.find(private_filter) + nodes_coll = nodes_coll.find({'project': {'$in': _public_project_ids()}}) node_count = nodes_coll.count() - log.debug('Reindexing %d in Elastic', node_count) + log.debug('Reindexing nodes %d in Elastic', node_count) from pillar.celery.search_index_tasks import prepare_node_data from pillar.api.search import elastic_indexing @@ -93,12 +88,12 @@ def _reindex_nodes(): try: to_index = prepare_node_data('', node=node) elastic_indexing.index_node_save(to_index) - except(KeyError, AttributeError): + except (KeyError, AttributeError): log.exception('Field is missing for %s', node) continue -@manager_elk.option('indexname', nargs='?') +@manager_elastic.option('indexname', nargs='?') def reindex(indexname=''): if not indexname: log.info('reindex everything..') diff --git a/pillar/config.py b/pillar/config.py index 1baf92d4..44e6598b 100644 --- a/pillar/config.py +++ b/pillar/config.py @@ -78,10 +78,11 @@ ALGOLIA_API_KEY = '-SECRET-' ALGOLIA_INDEX_USERS = 'dev_Users' ALGOLIA_INDEX_NODES = 'dev_Nodes' -SEARCH_BACKENDS = ['algolia', 'elastic'] +SEARCH_BACKENDS = ('algolia', 'elastic') # search backend we use -ELASTIC_SEARCH_HOSTS = ['elasticsearch'] +ELASTIC_SEARCH_HOSTS = ['elasticsearch'] # elasticsearch hosts ELASTIC_INDICES = { + # elasticsearch indexes 'NODE': 'nodes', 'USER': 'users', }