From 533544117bf3a09bc5f00680a6caeee671e964f9 Mon Sep 17 00:00:00 2001 From: Stephan Preeker Date: Fri, 8 Dec 2017 13:12:39 +0100 Subject: [PATCH] T52710 search on id works --- pillar/api/search/documents.py | 6 +++- pillar/api/search/elastic_indexing.py | 19 ++++-------- pillar/api/search/queries.py | 43 ++++++++++++++------------- pillar/cli/elastic.py | 40 +++++++++++++++++++++---- 4 files changed, 67 insertions(+), 41 deletions(-) diff --git a/pillar/api/search/documents.py b/pillar/api/search/documents.py index bb029df3..0c5e870c 100644 --- a/pillar/api/search/documents.py +++ b/pillar/api/search/documents.py @@ -115,7 +115,11 @@ def create_doc_from_user_data(user_to_index): def create_doc_from_node_data(node_to_index): # node stuff - doc_id = str(node_to_index['objectID']) + doc_id = str(node_to_index.get('objectID', '')) + + if not doc_id: + log.error('ID missing %s', node_to_index) + return doc = Node(_id=doc_id) diff --git a/pillar/api/search/elastic_indexing.py b/pillar/api/search/elastic_indexing.py index c0c88bbf..c57ec158 100644 --- a/pillar/api/search/elastic_indexing.py +++ b/pillar/api/search/elastic_indexing.py @@ -21,32 +21,23 @@ def push_updated_user(user_to_index: dict): Push an update to the Elastic index when a user item is updated. """ - - log.warning( - 'WIP USER ELK INDEXING %s %s', - user_to_index.get('username'), - user_to_index.get('objectID')) - doc = documents.create_doc_from_user_data(user_to_index) + log.debug('UPDATE USER %s', doc._id) doc.save() - log.warning('CREATED ELK USER DOC') def index_node_save(node_to_index: dict): - log.warning( - 'ELK NODE INDEXING %s', - node_to_index.get('objectID')) - - log.warning(node_to_index) + if not node_to_index: + return doc = documents.create_doc_from_node_data(node_to_index) - log.warning('CREATED ELK NODE DOC') + log.debug('CREATED ELK NODE DOC %s', doc._id) doc.save() def index_node_delete(delete_id: str): - log.warning('NODE DELETE INDEXING %s', delete_id) + log.debug('NODE DELETE INDEXING %s', delete_id) documents.Node(id=delete_id).delete() diff --git a/pillar/api/search/queries.py b/pillar/api/search/queries.py index f21f7ca4..a65641dc 100644 --- a/pillar/api/search/queries.py +++ b/pillar/api/search/queries.py @@ -22,29 +22,26 @@ def add_aggs_to_search(search, agg_terms): search.aggs.bucket(term, 'terms', field=term) -def make_must(terms): +def make_must(must: list, terms: dict) -> list: """ Given some term parameters we must match those """ - must = [] - for field, value in terms.items(): - print(field, value) - if value: must.append({'match': {field: value}}) return must -def nested_bool(should, terms): +def nested_bool(must: list, should: list, terms: dict) -> Search: """ + Create a nested bool, where the aggregation + selection is a must """ - must = [] - must = make_must(terms) + must = make_must(must, terms) bool_query = Q('bool', should=should) must.append(bool_query) bool_query = Q('bool', must=must) @@ -70,13 +67,14 @@ def do_search(query: str, terms: dict) -> dict: Q('term', tags=query), ] - if query: - search = nested_bool(should, terms) - else: - # do a match all for the aggregations - search = Search(using=client) - search.query = Q('term', _type='node') + must = [ + Q('term', _type='node') + ] + if not query: + should = [] + + search = nested_bool(must, should, terms) add_aggs_to_search(search, node_agg_terms) if current_app.config['DEBUG']: @@ -99,13 +97,18 @@ def do_user_search(query: str, terms: dict) -> dict: Q('match', full_name=query), ] - if query: - search = nested_bool(should, terms) - else: - # do a match all for the aggregations - search = Search(using=client) - search.query = Q('term', _type='user') + must = [ + Q('term', _type='user') + ] + # We got an id field. we MUST find it. + if len(query) == len('563aca02c379cf0005e8e17d'): + must.append(Q('term', _id=query)) + + if not query: + should = [] + + search = nested_bool(must, should, terms) add_aggs_to_search(search, user_agg_terms) if current_app.config['DEBUG']: diff --git a/pillar/cli/elastic.py b/pillar/cli/elastic.py index cee03f81..a5adeb73 100644 --- a/pillar/cli/elastic.py +++ b/pillar/cli/elastic.py @@ -48,12 +48,35 @@ def _reindex_users(): if not to_index: log.debug('missing user..') continue - elastic_indexing.push_updated_user(to_index) + + try: + elastic_indexing.push_updated_user(to_index) + except(KeyError, AttributeError): + log.exception('Field is missing for %s', user) + continue + + +# stolen from api.latest. +def _public_project_ids() -> typing.List[bson.ObjectId]: + """Returns a list of ObjectIDs of public projects. + + Memoized in setup_app(). + """ + + proj_coll = current_app.db('projects') + result = proj_coll.find({'is_private': False}, {'_id': 1}) + return [p['_id'] for p in result] def _reindex_nodes(): + db = current_app.db() + pipeline = [ + {'$match': {'project': {'$in': _public_project_ids()}}}, + ] + private_filter = {'project': {'$in': _public_project_ids()}} nodes_coll = db['nodes'] + nodes_coll = nodes_coll.find(private_filter) node_count = nodes_coll.count() log.debug('Reindexing %d in Elastic', node_count) @@ -61,20 +84,25 @@ def _reindex_nodes(): from pillar.celery.search_index_tasks import prepare_node_data from pillar.api.search import elastic_indexing - for node in nodes_coll.find(): - to_index = prepare_node_data('', node=node) - elastic_indexing.index_node_save(to_index) + for node in nodes_coll: + try: + to_index = prepare_node_data('', node=node) + elastic_indexing.index_node_save(to_index) + except(KeyError, AttributeError): + log.exception('Field is missing for %s', node) + continue @manager_elk.command -def reindex(indexname=None): +def reindex(indexname): if not indexname: log.debug('reindex everything..') _reindex_nodes() _reindex_users() - elif indexname == 'users': + log.debug('Indexing %s', indexname) _reindex_users() elif indexname == 'nodes': + log.debug('Indexing %s', indexname) _reindex_nodes()