archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Elastic reindexing: more verbose logging at info level.

Browse Source 
Including some progress report every 100 nodes.
This commit is contained in:
Sybren A. Stüvel 2018-01-05 14:21:39 +01:00
parent bbdb731043
commit 1bfda7769a
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pillar/cli/elastic.py
View File

@ -76,21 +76,26 @@ def _public_project_ids() -> typing.List[bson.ObjectId]:
def _reindex_nodes():
db = current_app.db()
nodes_coll = db['nodes']
nodes_coll = nodes_coll.find({'project': {'$in': _public_project_ids()}})
node_count = nodes_coll.count()
nodes = nodes_coll.find({'project': {'$in': _public_project_ids()}})
node_count = nodes.count()
log.debug('Nodes %d will be reindexed in Elastic', node_count)
log.info('Nodes %d will be reindexed in Elastic', node_count)
from pillar.celery.search_index_tasks import prepare_node_data
from pillar.api.search import elastic_indexing
for node in nodes_coll:
indexed = 0
for idx, node in enumerate(nodes):
if idx % 100 == 0:
log.info('Processing node %d/%d', idx+1, node_count)
try:
to_index = prepare_node_data('', node=node)
elastic_indexing.index_node_save(to_index)
except (KeyError, AttributeError):
log.exception('%s is missing Field', node)
continue
log.exception('Node %s is missing Field', node)
else:
indexed += 1
log.info('Reindexed %d/%d nodes', indexed, node_count)
@manager_elastic.option('indexname', nargs='?')