T52710 search on id works

pillar/api/search/documents.py

@@ -115,7 +115,11 @@ def create_doc_from_user_data(user_to_index):
 def create_doc_from_node_data(node_to_index):
 
     # node stuff
-    doc_id = str(node_to_index['objectID'])
+    doc_id = str(node_to_index.get('objectID', ''))
+
+    if not doc_id:
+        log.error('ID missing %s', node_to_index)
+        return
 
     doc = Node(_id=doc_id)
 

pillar/api/search/elastic_indexing.py

@@ -21,32 +21,23 @@ def push_updated_user(user_to_index: dict):
     Push an update to the Elastic index when
     a user item is updated.
     """
-
-    log.warning(
-        'WIP USER ELK INDEXING %s %s',
-        user_to_index.get('username'),
-        user_to_index.get('objectID'))
-
     doc = documents.create_doc_from_user_data(user_to_index)
+    log.debug('UPDATE USER %s', doc._id)
     doc.save()
-    log.warning('CREATED ELK USER DOC')
 
 
 def index_node_save(node_to_index: dict):
 
-    log.warning(
-        'ELK NODE INDEXING %s',
-        node_to_index.get('objectID'))
-
-    log.warning(node_to_index)
+    if not node_to_index:
+        return
 
     doc = documents.create_doc_from_node_data(node_to_index)
 
-    log.warning('CREATED ELK NODE DOC')
+    log.debug('CREATED ELK NODE DOC %s', doc._id)
     doc.save()
 
 
 def index_node_delete(delete_id: str):
 
-    log.warning('NODE DELETE INDEXING %s', delete_id)
+    log.debug('NODE DELETE INDEXING %s', delete_id)
     documents.Node(id=delete_id).delete()

pillar/api/search/queries.py

@@ -22,29 +22,26 @@ def add_aggs_to_search(search, agg_terms):
         search.aggs.bucket(term, 'terms', field=term)
 
 
-def make_must(terms):
+def make_must(must: list, terms: dict) -> list:
     """
     Given some term parameters
     we must match those
     """
 
-    must = []
-
     for field, value in terms.items():
 
-        print(field, value)
-
         if value:
             must.append({'match': {field: value}})
 
     return must
 
 
-def nested_bool(should, terms):
+def nested_bool(must: list, should: list, terms: dict) -> Search:
     """
+    Create a nested bool, where the aggregation
+    selection is a must
     """
-    must = []
-    must = make_must(terms)
+    must = make_must(must, terms)
     bool_query = Q('bool', should=should)
     must.append(bool_query)
     bool_query = Q('bool', must=must)
@@ -70,13 +67,14 @@ def do_search(query: str, terms: dict) -> dict:
         Q('term', tags=query),
     ]
 
-    if query:
-        search = nested_bool(should, terms)
-    else:
-        # do a match all for the aggregations
-        search = Search(using=client)
-        search.query = Q('term', _type='node')
+    must = [
+        Q('term', _type='node')
+    ]
 
+    if not query:
+        should = []
+
+    search = nested_bool(must, should, terms)
     add_aggs_to_search(search, node_agg_terms)
 
     if current_app.config['DEBUG']:
@@ -99,13 +97,18 @@ def do_user_search(query: str, terms: dict) -> dict:
         Q('match', full_name=query),
     ]
 
-    if query:
-        search = nested_bool(should, terms)
-    else:
-        # do a match all for the aggregations
-        search = Search(using=client)
-        search.query = Q('term', _type='user')
+    must = [
+        Q('term', _type='user')
+    ]
 
+    # We got an id field. we MUST find it.
+    if len(query) == len('563aca02c379cf0005e8e17d'):
+        must.append(Q('term', _id=query))
+
+    if not query:
+        should = []
+
+    search = nested_bool(must, should, terms)
     add_aggs_to_search(search, user_agg_terms)
 
     if current_app.config['DEBUG']:

pillar/cli/elastic.py

@@ -48,12 +48,35 @@ def _reindex_users():
         if not to_index:
             log.debug('missing user..')
             continue
-        elastic_indexing.push_updated_user(to_index)
+
+        try:
+            elastic_indexing.push_updated_user(to_index)
+        except(KeyError, AttributeError):
+            log.exception('Field is missing for %s', user)
+            continue
+
+
+# stolen from api.latest.
+def _public_project_ids() -> typing.List[bson.ObjectId]:
+    """Returns a list of ObjectIDs of public projects.
+
+    Memoized in setup_app().
+    """
+
+    proj_coll = current_app.db('projects')
+    result = proj_coll.find({'is_private': False}, {'_id': 1})
+    return [p['_id'] for p in result]
 
 
 def _reindex_nodes():
+
     db = current_app.db()
+    pipeline = [
+        {'$match': {'project': {'$in': _public_project_ids()}}},
+    ]
+    private_filter = {'project': {'$in': _public_project_ids()}}
     nodes_coll = db['nodes']
+    nodes_coll = nodes_coll.find(private_filter)
     node_count = nodes_coll.count()
 
     log.debug('Reindexing %d in Elastic', node_count)
@@ -61,20 +84,25 @@ def _reindex_nodes():
     from pillar.celery.search_index_tasks import prepare_node_data
     from pillar.api.search import elastic_indexing
 
-    for node in nodes_coll.find():
-        to_index = prepare_node_data('', node=node)
-        elastic_indexing.index_node_save(to_index)
+    for node in nodes_coll:
+        try:
+            to_index = prepare_node_data('', node=node)
+            elastic_indexing.index_node_save(to_index)
+        except(KeyError, AttributeError):
+            log.exception('Field is missing for %s', node)
+            continue
 
 
 @manager_elk.command
-def reindex(indexname=None):
+def reindex(indexname):
 
     if not indexname:
         log.debug('reindex everything..')
         _reindex_nodes()
         _reindex_users()
-
     elif indexname == 'users':
+        log.debug('Indexing %s', indexname)
         _reindex_users()
     elif indexname == 'nodes':
+        log.debug('Indexing %s', indexname)
         _reindex_nodes()