diff --git a/pillar/api/search/documents.py b/pillar/api/search/documents.py index 0c3d5249..bb029df3 100644 --- a/pillar/api/search/documents.py +++ b/pillar/api/search/documents.py @@ -28,7 +28,22 @@ autocomplete = es.analyzer( class User(es.DocType): """Elastic document describing user.""" - name = es.String( + objectID = es.Keyword() + + username = es.String( + fielddata=True, + analyzer=autocomplete, + ) + + full_name = es.String( + fielddata=True, + analyzer=autocomplete, + ) + + roles = es.Keyword(multi=True) + groups = es.Keyword(multi=True) + + email = es.String( fielddata=True, analyzer=autocomplete, ) @@ -86,8 +101,14 @@ class Node(es.DocType): def create_doc_from_user_data(user_to_index): - doc_id = user_to_index['objectID'] + doc_id = str(user_to_index['objectID']) doc = User(_id=doc_id) + doc.objectID = str(user_to_index['objectID']) + doc.username = user_to_index['username'] + doc.full_name = user_to_index['full_name'] + doc.roles = list(map(str, user_to_index['roles'])) + doc.groups = list(map(str, user_to_index['groups'])) + doc.email = user_to_index['email'] return doc @@ -95,8 +116,10 @@ def create_doc_from_node_data(node_to_index): # node stuff doc_id = str(node_to_index['objectID']) + doc = Node(_id=doc_id) + doc.objectID = str(node_to_index['objectID']) doc.node_type = node_to_index['node_type'] doc.name = node_to_index['name'] doc.user.id = str(node_to_index['user']['_id']) @@ -116,3 +139,20 @@ def create_doc_from_node_data(node_to_index): doc.updated_at = node_to_index['updated'] return doc + + +def create_doc_from_user(user_to_index: dict) -> User: + """ + Create a user document from user + """ + + doc_id = str(user_to_index['objectID']) + doc = User(_id=doc_id) + doc.objectID = str(user_to_index['objectID']) + doc.full_name = user_to_index['full_name'] + doc.username = user_to_index['username'] + doc.roles = user_to_index['roles'] + doc.groups = user_to_index['groups'] + doc.email = user_to_index['email'] + + return doc diff --git a/pillar/api/search/elastic_indexing.py b/pillar/api/search/elastic_indexing.py index a76f0fa3..1e372087 100644 --- a/pillar/api/search/elastic_indexing.py +++ b/pillar/api/search/elastic_indexing.py @@ -26,6 +26,10 @@ def push_updated_user(user_to_index: dict): user_to_index.get('username'), user_to_index.get('objectID')) + doc = documents.create_doc_from_user_data(user_to_index) + doc.save() + log.warning('CREATED ELK USER DOC') + def index_node_save(node_to_index: dict): @@ -37,7 +41,7 @@ def index_node_save(node_to_index: dict): doc = documents.create_doc_from_node_data(node_to_index) - log.warning('CREATED ELK DOC') + log.warning('CREATED ELK NODE DOC') doc.save() diff --git a/pillar/api/search/index.py b/pillar/api/search/index.py index d052f580..0ecc70d8 100644 --- a/pillar/api/search/index.py +++ b/pillar/api/search/index.py @@ -64,6 +64,21 @@ class ResetNodeIndex(ResetIndexTask): doc_types = [documents.Node] +class ResetUserIndex(ResetIndexTask): + index = current_app.config['ELASTIC_INDICES']['USER'] + doc_types = [documents.User] + + def reset_node_index(): resettask = ResetNodeIndex() resettask.execute() + + +def reset_index(indexnames): + if 'users' in indexnames: + resettask = ResetUserIndex() + resettask.execute() + if 'nodes' in indexnames: + resettask = ResetUserIndex() + resettask.execute() + diff --git a/pillar/api/search/queries.py b/pillar/api/search/queries.py index 26d7698d..2fe038be 100644 --- a/pillar/api/search/queries.py +++ b/pillar/api/search/queries.py @@ -42,3 +42,44 @@ def do_search(query: str) -> dict: response = search.execute() return response.to_dict() + + +def do_user_search(query: str) -> dict: + """ + return user objects + """ + should = [ + Q('match', username=query), + Q('match', full_name=query), + ] + bool_query = Q('bool', should=should) + search = Search(using=client) + search.query = bool_query + + if current_app.config['DEBUG']: + log.debug(json.dumps(search.to_dict(), indent=4)) + + response = search.execute() + + return response.to_dict() + + +def do_user_search_admin(query: str) -> dict: + """ + return users with all fields and aggregations + """ + should = [ + Q('match', username=query), + Q('match', email=query), + Q('match', full_name=query), + ] + bool_query = Q('bool', should=should) + search = Search(using=client) + search.query = bool_query + + if current_app.config['DEBUG']: + log.debug(json.dumps(search.to_dict(), indent=4)) + + response = search.execute() + + return response.to_dict() diff --git a/pillar/api/search/routes.py b/pillar/api/search/routes.py index 77f1a7e7..99a18be8 100644 --- a/pillar/api/search/routes.py +++ b/pillar/api/search/routes.py @@ -18,16 +18,58 @@ blueprint_search = Blueprint('elksearch', __name__) from . import queries -#@authorization.require_login(require_cap='subscriber') -@blueprint_search.route('/', methods=['GET']) -def search_nodes(): + +def _valid_search() -> [str, str]: + """ + Validate search parameters + """ searchword = request.args.get('q', '') if not searchword: - return 'You are forgetting a "?q=whatareyoulookingfor"' + return '', 'You are forgetting a "?q=whatareyoulookingfor"' + + return searchword, '' + + +@blueprint_search.route('/', methods=['GET']) +def search_nodes(): + + searchword, err = _valid_search() + if err: + return err data = queries.do_search(searchword) resp = Response(json.dumps(data), mimetype='application/json') return resp + + +@blueprint_search.route('/user', methods=['GET']) +def search_user(): + + searchword, err = _valid_search() + if err: + return err + + data = queries.do_user_search(searchword) + + resp = Response(json.dumps(data), mimetype='application/json') + return resp + + +@authorization.require_login(require_cap='admin') +@blueprint_search.route('/admin/user', methods=['GET']) +def search_user_admin(): + """ + User search over all fields. + """ + + searchword, err = _valid_search() + if err: + return err + + data = queries.do_user_search_admin(searchword) + + resp = Response(json.dumps(data), mimetype='application/json') + return resp diff --git a/pillar/celery/search_index_tasks.py b/pillar/celery/search_index_tasks.py index e5cac649..2c045e56 100644 --- a/pillar/celery/search_index_tasks.py +++ b/pillar/celery/search_index_tasks.py @@ -117,20 +117,23 @@ def prepare_node_data(node_id: str, node=None) -> dict: return to_index -def prepare_user_data(user_id: str) -> dict: +def prepare_user_data(user_id: str, user=None) -> dict: """ Prepare data to index for user node """ - user_oid = ObjectId(user_id) - log.info('Retrieving user %s', user_oid) - users_coll = current_app.db('users') - user = users_coll.find_one({'_id': user_oid}) + if not user: + user_oid = ObjectId(user_id) + log.info('Retrieving user %s', user_oid) + users_coll = current_app.db('users') + user = users_coll.find_one({'_id': user_oid}) + if user is None: log.warning('Unable to find user %s, not updating Algolia.', user_oid) return user_roles = set(user.get('roles', ())) + if 'service' in user_roles: return diff --git a/pillar/cli/elastic.py b/pillar/cli/elastic.py index dcadc55f..cee03f81 100644 --- a/pillar/cli/elastic.py +++ b/pillar/cli/elastic.py @@ -9,25 +9,49 @@ log = logging.getLogger(__name__) manager_elk = Manager( current_app, usage="Elastic utilities, like reset_index()") +indexes = ['users', 'nodes'] + @manager_elk.command -def reset_index(elk_index): +def reset_index(elk_index=None): """ Destroy and recreate elastic indices node, user ... """ - #real_current_app = current_app._get_current_object()._get_current_object() with current_app.app_context(): from pillar.api.search import index + if not elk_index: + index.reset_index(indexes) + return if elk_index == 'nodes': - index.reset_node_index() + index.reset_index(['node']) + return + if elk_index == 'users': + index.reset_index(['user']) + return -@manager_elk.command -def reindex_nodes(): +def _reindex_users(): + db = current_app.db() + users_coll = db['users'] + user_count = users_coll.count() + log.debug('Reindexing %d in Elastic', user_count) + + from pillar.celery.search_index_tasks import prepare_user_data + from pillar.api.search import elastic_indexing + + for user in users_coll.find(): + to_index = prepare_user_data('', user=user) + if not to_index: + log.debug('missing user..') + continue + elastic_indexing.push_updated_user(to_index) + + +def _reindex_nodes(): db = current_app.db() nodes_coll = db['nodes'] node_count = nodes_coll.count() @@ -40,3 +64,17 @@ def reindex_nodes(): for node in nodes_coll.find(): to_index = prepare_node_data('', node=node) elastic_indexing.index_node_save(to_index) + + +@manager_elk.command +def reindex(indexname=None): + + if not indexname: + log.debug('reindex everything..') + _reindex_nodes() + _reindex_users() + + elif indexname == 'users': + _reindex_users() + elif indexname == 'nodes': + _reindex_nodes() diff --git a/src/scripts/tutti/4_search.js b/src/scripts/tutti/4_search.js index 63ae6834..b73068ff 100644 --- a/src/scripts/tutti/4_search.js +++ b/src/scripts/tutti/4_search.js @@ -30,7 +30,6 @@ $(document).ready(function() { var tu = searchInput.typeahead({hint: true}, { //source: algoliaIndex.ttAdapter(), source: elasticSearch(), - //source: elkBlood(), async: true, displayKey: 'name', limit: 10,