archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Merge branch 'elastic' of git.blender.org:pillar into elastic

Browse Source
This commit is contained in:
Stephan preeker 2017-12-08 17:09:11 +01:00
commit 8206186426
2 changed files with 33 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pillar/api/search/index.py
View File

@ -21,13 +21,15 @@ class ResetIndexTask(object):
""" """
Clear and build index / mapping Clear and build index / mapping
""" """
index = '' index_key = ''
"""Key into the ELASTIC_INDICES dict in the app config."""
doc_types = [] doc_types = []
name = 'remove index' name = 'remove index'
def __init__(self): def __init__(self):
if not self.index: if not self.index_key:
raise ValueError("No index specified") raise ValueError("No index specified")
if not self.doc_types: if not self.doc_types:
@ -40,16 +42,16 @@ class ResetIndexTask(object):
) )
def execute(self): def execute(self):
index = current_app.config['ELASTIC_INDICES'][self.index_key]
idx = es.Index(self.index) idx = es.Index(index)
try: try:
idx.delete(ignore=404) idx.delete(ignore=404)
log.info("Deleted index %s", self.index) log.info("Deleted index %s", index)
except AttributeError: except AttributeError:
log.warning("Could not delete index '%s', ignoring", self.index) log.warning("Could not delete index '%s', ignoring", index)
except NotFoundError: except NotFoundError:
log.warning("Could not delete index '%s', ignoring", self.index) log.warning("Could not delete index '%s', ignoring", index)
# create doc types # create doc types
for dt in self.doc_types: for dt in self.doc_types:
@ -60,25 +62,10 @@ class ResetIndexTask(object):
class ResetNodeIndex(ResetIndexTask): class ResetNodeIndex(ResetIndexTask):
index = current_app.config['ELASTIC_INDICES']['NODE'] index_key = 'NODE'
doc_types = [documents.Node] doc_types = [documents.Node]
class ResetUserIndex(ResetIndexTask): class ResetUserIndex(ResetIndexTask):
index = current_app.config['ELASTIC_INDICES']['USER'] index_key = 'USER'
doc_types = [documents.User] doc_types = [documents.User]
def reset_node_index():
resettask = ResetNodeIndex()
resettask.execute()
def reset_index(indexnames):
if 'users' in indexnames:
resettask = ResetUserIndex()
resettask.execute()
if 'nodes' in indexnames:
resettask = ResetUserIndex()
resettask.execute()

42
pillar/cli/elastic.py
View File

@ -5,17 +5,21 @@ import bson
from flask_script import Manager from flask_script import Manager
from pillar import current_app from pillar import current_app
from pillar.api.search import index
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
manager_elk = Manager( manager_elk = Manager(
current_app, usage="Elastic utilities, like reset_index()") current_app, usage="Elastic utilities, like reset_index()")
indexes = ['users', 'nodes'] name_to_task = {
'nodes': index.ResetNodeIndex,
'users': index.ResetUserIndex,
}
@manager_elk.command @manager_elk.option('indices', nargs='*')
def reset_index(elk_index=None): def reset_index(indices):
""" """
Destroy and recreate elastic indices Destroy and recreate elastic indices
@ -23,16 +27,16 @@ def reset_index(elk_index=None):
""" """
with current_app.app_context(): with current_app.app_context():
from pillar.api.search import index if not indices:
if not elk_index: indices = name_to_task.keys()
index.reset_index(indexes)
return for elk_index in indices:
if elk_index == 'nodes': try:
index.reset_index(['node']) task = name_to_task[elk_index]()
return except KeyError:
if elk_index == 'users': raise SystemError('Unknown elk_index, choose from %s' %
index.reset_index(['user']) (', '.join(name_to_task.keys())))
return task.execute()
def _reindex_users(): def _reindex_users():
@ -71,7 +75,6 @@ def _public_project_ids() -> typing.List[bson.ObjectId]:
def _reindex_nodes(): def _reindex_nodes():
db = current_app.db() db = current_app.db()
pipeline = [ pipeline = [
{'$match': {'project': {'$in': _public_project_ids()}}}, {'$match': {'project': {'$in': _public_project_ids()}}},
@ -95,16 +98,15 @@ def _reindex_nodes():
continue continue
@manager_elk.command @manager_elk.option('indexname', nargs='?')
def reindex(indexname): def reindex(indexname=''):
if not indexname: if not indexname:
log.debug('reindex everything..') log.info('reindex everything..')
_reindex_nodes() _reindex_nodes()
_reindex_users() _reindex_users()
elif indexname == 'users': elif indexname == 'users':
log.debug('Indexing %s', indexname) log.info('Indexing %s', indexname)
_reindex_users() _reindex_users()
elif indexname == 'nodes': elif indexname == 'nodes':
log.debug('Indexing %s', indexname) log.info('Indexing %s', indexname)
_reindex_nodes() _reindex_nodes()