T53161 Proof of Concept working USER search. WIP js.

pillar/api/search/documents.py

@@ -28,7 +28,22 @@ autocomplete = es.analyzer(
 class User(es.DocType):
     """Elastic document describing user."""
 
-    name = es.String(
+    objectID = es.Keyword()
+
+    username = es.String(
+        fielddata=True,
+        analyzer=autocomplete,
+    )
+
+    full_name = es.String(
+        fielddata=True,
+        analyzer=autocomplete,
+    )
+
+    roles = es.Keyword(multi=True)
+    groups = es.Keyword(multi=True)
+
+    email = es.String(
         fielddata=True,
         analyzer=autocomplete,
     )
@@ -86,8 +101,14 @@ class Node(es.DocType):
 
 
 def create_doc_from_user_data(user_to_index):
-    doc_id = user_to_index['objectID']
+    doc_id = str(user_to_index['objectID'])
     doc = User(_id=doc_id)
+    doc.objectID = str(user_to_index['objectID'])
+    doc.username = user_to_index['username']
+    doc.full_name = user_to_index['full_name']
+    doc.roles = list(map(str, user_to_index['roles']))
+    doc.groups = list(map(str, user_to_index['groups']))
+    doc.email = user_to_index['email']
     return doc
 
 
@@ -95,8 +116,10 @@ def create_doc_from_node_data(node_to_index):
 
     # node stuff
     doc_id = str(node_to_index['objectID'])
+
     doc = Node(_id=doc_id)
 
+    doc.objectID = str(node_to_index['objectID'])
     doc.node_type = node_to_index['node_type']
     doc.name = node_to_index['name']
     doc.user.id = str(node_to_index['user']['_id'])
@@ -116,3 +139,20 @@ def create_doc_from_node_data(node_to_index):
     doc.updated_at = node_to_index['updated']
 
     return doc
+
+
+def create_doc_from_user(user_to_index: dict) -> User:
+    """
+    Create a user document from user
+    """
+
+    doc_id = str(user_to_index['objectID'])
+    doc = User(_id=doc_id)
+    doc.objectID = str(user_to_index['objectID'])
+    doc.full_name = user_to_index['full_name']
+    doc.username = user_to_index['username']
+    doc.roles = user_to_index['roles']
+    doc.groups = user_to_index['groups']
+    doc.email = user_to_index['email']
+
+    return doc

pillar/api/search/elastic_indexing.py

@@ -26,6 +26,10 @@ def push_updated_user(user_to_index: dict):
         user_to_index.get('username'),
         user_to_index.get('objectID'))
 
+    doc = documents.create_doc_from_user_data(user_to_index)
+    doc.save()
+    log.warning('CREATED ELK USER DOC')
+
 
 def index_node_save(node_to_index: dict):
 
@@ -37,7 +41,7 @@ def index_node_save(node_to_index: dict):
 
     doc = documents.create_doc_from_node_data(node_to_index)
 
-    log.warning('CREATED ELK DOC')
+    log.warning('CREATED ELK NODE DOC')
     doc.save()
 
 

pillar/api/search/index.py

@@ -64,6 +64,21 @@ class ResetNodeIndex(ResetIndexTask):
     doc_types = [documents.Node]
 
 
+class ResetUserIndex(ResetIndexTask):
+    index = current_app.config['ELASTIC_INDICES']['USER']
+    doc_types = [documents.User]
+
+
 def reset_node_index():
     resettask = ResetNodeIndex()
     resettask.execute()
+
+
+def reset_index(indexnames):
+    if 'users' in indexnames:
+        resettask = ResetUserIndex()
+        resettask.execute()
+    if 'nodes' in indexnames:
+        resettask = ResetUserIndex()
+        resettask.execute()
+

pillar/api/search/queries.py

@@ -42,3 +42,44 @@ def do_search(query: str) -> dict:
     response = search.execute()
 
     return response.to_dict()
+
+
+def do_user_search(query: str) -> dict:
+    """
+    return user objects
+    """
+    should = [
+        Q('match', username=query),
+        Q('match', full_name=query),
+    ]
+    bool_query = Q('bool', should=should)
+    search = Search(using=client)
+    search.query = bool_query
+
+    if current_app.config['DEBUG']:
+        log.debug(json.dumps(search.to_dict(), indent=4))
+
+    response = search.execute()
+
+    return response.to_dict()
+
+
+def do_user_search_admin(query: str) -> dict:
+    """
+    return users with all fields and aggregations
+    """
+    should = [
+        Q('match', username=query),
+        Q('match', email=query),
+        Q('match', full_name=query),
+    ]
+    bool_query = Q('bool', should=should)
+    search = Search(using=client)
+    search.query = bool_query
+
+    if current_app.config['DEBUG']:
+        log.debug(json.dumps(search.to_dict(), indent=4))
+
+    response = search.execute()
+
+    return response.to_dict()

pillar/api/search/routes.py

@@ -18,16 +18,58 @@ blueprint_search = Blueprint('elksearch', __name__)
 
 from . import queries
 
-#@authorization.require_login(require_cap='subscriber')
-@blueprint_search.route('/', methods=['GET'])
-def search_nodes():
+
+def _valid_search() -> [str, str]:
+    """
+    Validate search parameters
+    """
 
     searchword = request.args.get('q', '')
 
     if not searchword:
-        return 'You are forgetting a "?q=whatareyoulookingfor"'
+        return '', 'You are forgetting a "?q=whatareyoulookingfor"'
+
+    return searchword, ''
+
+
+@blueprint_search.route('/', methods=['GET'])
+def search_nodes():
+
+    searchword, err = _valid_search()
+    if err:
+        return err
 
     data = queries.do_search(searchword)
 
     resp = Response(json.dumps(data), mimetype='application/json')
     return resp
+
+
+@blueprint_search.route('/user', methods=['GET'])
+def search_user():
+
+    searchword, err = _valid_search()
+    if err:
+        return err
+
+    data = queries.do_user_search(searchword)
+
+    resp = Response(json.dumps(data), mimetype='application/json')
+    return resp
+
+
+@authorization.require_login(require_cap='admin')
+@blueprint_search.route('/admin/user', methods=['GET'])
+def search_user_admin():
+    """
+    User search over all fields.
+    """
+
+    searchword, err = _valid_search()
+    if err:
+        return err
+
+    data = queries.do_user_search_admin(searchword)
+
+    resp = Response(json.dumps(data), mimetype='application/json')
+    return resp

pillar/celery/search_index_tasks.py

@@ -117,20 +117,23 @@ def prepare_node_data(node_id: str, node=None) -> dict:
     return to_index
 
 
-def prepare_user_data(user_id: str) -> dict:
+def prepare_user_data(user_id: str, user=None) -> dict:
     """
     Prepare data to index for user node
     """
 
+    if not user:
         user_oid = ObjectId(user_id)
         log.info('Retrieving user %s', user_oid)
         users_coll = current_app.db('users')
         user = users_coll.find_one({'_id': user_oid})
+
     if user is None:
         log.warning('Unable to find user %s, not updating Algolia.', user_oid)
         return
 
     user_roles = set(user.get('roles', ()))
+
     if 'service' in user_roles:
         return
 

pillar/cli/elastic.py

@@ -9,25 +9,49 @@ log = logging.getLogger(__name__)
 manager_elk = Manager(
     current_app, usage="Elastic utilities, like reset_index()")
 
+indexes = ['users', 'nodes']
+
 
 @manager_elk.command
-def reset_index(elk_index):
+def reset_index(elk_index=None):
     """
     Destroy and recreate elastic indices
 
     node, user ...
     """
-    #real_current_app = current_app._get_current_object()._get_current_object()
 
     with current_app.app_context():
         from pillar.api.search import index
+        if not elk_index:
+            index.reset_index(indexes)
+            return
         if elk_index == 'nodes':
-            index.reset_node_index()
+            index.reset_index(['node'])
+            return
+        if elk_index == 'users':
+            index.reset_index(['user'])
+            return
 
 
-@manager_elk.command
-def reindex_nodes():
+def _reindex_users():
+    db = current_app.db()
+    users_coll = db['users']
+    user_count = users_coll.count()
 
+    log.debug('Reindexing %d in Elastic', user_count)
+
+    from pillar.celery.search_index_tasks import prepare_user_data
+    from pillar.api.search import elastic_indexing
+
+    for user in users_coll.find():
+        to_index = prepare_user_data('', user=user)
+        if not to_index:
+            log.debug('missing user..')
+            continue
+        elastic_indexing.push_updated_user(to_index)
+
+
+def _reindex_nodes():
     db = current_app.db()
     nodes_coll = db['nodes']
     node_count = nodes_coll.count()
@@ -40,3 +64,17 @@ def reindex_nodes():
     for node in nodes_coll.find():
         to_index = prepare_node_data('', node=node)
         elastic_indexing.index_node_save(to_index)
+
+
+@manager_elk.command
+def reindex(indexname=None):
+
+    if not indexname:
+        log.debug('reindex everything..')
+        _reindex_nodes()
+        _reindex_users()
+
+    elif indexname == 'users':
+        _reindex_users()
+    elif indexname == 'nodes':
+        _reindex_nodes()

src/scripts/tutti/4_search.js

@@ -30,7 +30,6 @@ $(document).ready(function() {
 	var tu = searchInput.typeahead({hint: true}, {
 		//source: algoliaIndex.ttAdapter(),
 		source: elasticSearch(),
-		//source: elkBlood(),
 		async: true,
 		displayKey: 'name',
 		limit: 10,