archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

wip D2950

Browse Source
This commit is contained in:
Stephan preeker 2017-12-15 17:57:47 +01:00
parent 7c6425ff4d
commit 10732f9a10
5 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pillar/api/search/elastic_indexing.py
View File

@ -12,7 +12,7 @@ elk_hosts = current_app.config['ELASTIC_SEARCH_HOSTS']
connections.create_connection( connections.create_connection(
hosts=elk_hosts, hosts=elk_hosts,
sniff_on_start=True, sniff_on_start=False,
timeout=20) timeout=20)

3
pillar/api/search/index.py
View File

@ -17,7 +17,8 @@ class ResetIndexTask(object):
index_key = '' index_key = ''
"""Key into the ELASTIC_INDICES dict in the app config.""" """Key into the ELASTIC_INDICES dict in the app config."""
doc_types: List[type] = [] # List of elastic document types
doc_types = []
name = 'remove index' name = 'remove index'
def __init__(self): def __init__(self):

4
pillar/cli/__init__.py
View File

@ -12,7 +12,7 @@ from pillar.cli.celery import manager_celery
from pillar.cli.maintenance import manager_maintenance from pillar.cli.maintenance import manager_maintenance
from pillar.cli.operations import manager_operations from pillar.cli.operations import manager_operations
from pillar.cli.setup import manager_setup from pillar.cli.setup import manager_setup
from pillar.cli.elastic import manager_elk from pillar.cli.elastic import manager_elastic
from pillar.cli import translations from pillar.cli import translations
@ -23,4 +23,4 @@ manager.add_command('celery', manager_celery)
manager.add_command("maintenance", manager_maintenance) manager.add_command("maintenance", manager_maintenance)
manager.add_command("setup", manager_setup) manager.add_command("setup", manager_setup)
manager.add_command("operations", manager_operations) manager.add_command("operations", manager_operations)
manager.add_command("elastic", manager_elk) manager.add_command("elastic", manager_elastic)

23
pillar/cli/elastic.py
View File

@ -9,8 +9,8 @@ from pillar.api.search import index
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
manager_elk = Manager( manager_elastic = Manager(
current_app, usage="Elastic utilities, like reset_index()") current_app, usage="Elastic utilities")
name_to_task = { name_to_task = {
'nodes': index.ResetNodeIndex, 'nodes': index.ResetNodeIndex,
@ -18,12 +18,12 @@ name_to_task = {
} }
@manager_elk.option('indices', nargs='*') @manager_elastic.option('indices', nargs='*')
def reset_index(indices): def reset_index(indices):
""" """
Destroy and recreate elastic indices Destroy and recreate elastic indices
node, user ... nodes, users
""" """
with current_app.app_context(): with current_app.app_context():
@ -44,7 +44,7 @@ def _reindex_users():
users_coll = db['users'] users_coll = db['users']
user_count = users_coll.count() user_count = users_coll.count()
log.debug('Reindexing %d in Elastic', user_count) log.debug('Reindexing users %d in Elastic', user_count)
from pillar.celery.search_index_tasks import prepare_user_data from pillar.celery.search_index_tasks import prepare_user_data
from pillar.api.search import elastic_indexing from pillar.api.search import elastic_indexing
@ -62,7 +62,6 @@ def _reindex_users():
continue continue
# stolen from api.latest.
def _public_project_ids() -> typing.List[bson.ObjectId]: def _public_project_ids() -> typing.List[bson.ObjectId]:
"""Returns a list of ObjectIDs of public projects. """Returns a list of ObjectIDs of public projects.
@ -76,15 +75,11 @@ def _public_project_ids() -> typing.List[bson.ObjectId]:
def _reindex_nodes(): def _reindex_nodes():
db = current_app.db() db = current_app.db()
pipeline = [
{'$match': {'project': {'$in': _public_project_ids()}}},
]
private_filter = {'project': {'$in': _public_project_ids()}}
nodes_coll = db['nodes'] nodes_coll = db['nodes']
nodes_coll = nodes_coll.find(private_filter) nodes_coll = nodes_coll.find({'project': {'$in': _public_project_ids()}})
node_count = nodes_coll.count() node_count = nodes_coll.count()
log.debug('Reindexing %d in Elastic', node_count) log.debug('Reindexing nodes %d in Elastic', node_count)
from pillar.celery.search_index_tasks import prepare_node_data from pillar.celery.search_index_tasks import prepare_node_data
from pillar.api.search import elastic_indexing from pillar.api.search import elastic_indexing
@ -93,12 +88,12 @@ def _reindex_nodes():
try: try:
to_index = prepare_node_data('', node=node) to_index = prepare_node_data('', node=node)
elastic_indexing.index_node_save(to_index) elastic_indexing.index_node_save(to_index)
except(KeyError, AttributeError): except (KeyError, AttributeError):
log.exception('Field is missing for %s', node) log.exception('Field is missing for %s', node)
continue continue
@manager_elk.option('indexname', nargs='?') @manager_elastic.option('indexname', nargs='?')
def reindex(indexname=''): def reindex(indexname=''):
if not indexname: if not indexname:
log.info('reindex everything..') log.info('reindex everything..')

5
pillar/config.py
View File

@ -78,10 +78,11 @@ ALGOLIA_API_KEY = '-SECRET-'
ALGOLIA_INDEX_USERS = 'dev_Users' ALGOLIA_INDEX_USERS = 'dev_Users'
ALGOLIA_INDEX_NODES = 'dev_Nodes' ALGOLIA_INDEX_NODES = 'dev_Nodes'
SEARCH_BACKENDS = ['algolia', 'elastic'] SEARCH_BACKENDS = ('algolia', 'elastic') # search backend we use
ELASTIC_SEARCH_HOSTS = ['elasticsearch'] ELASTIC_SEARCH_HOSTS = ['elasticsearch'] # elasticsearch hosts
ELASTIC_INDICES = { ELASTIC_INDICES = {
# elasticsearch indexes
'NODE': 'nodes', 'NODE': 'nodes',
'USER': 'users', 'USER': 'users',
} }