From 1bfda7769a0c07a7e01efb0e6a464fd0a15c0456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= <sybren@stuvel.eu>
Date: Fri, 5 Jan 2018 14:21:39 +0100
Subject: [PATCH] Elastic reindexing: more verbose logging at info level.

Including some progress report every 100 nodes.
---
 pillar/cli/elastic.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/pillar/cli/elastic.py b/pillar/cli/elastic.py
index d659178f..e792e576 100644
--- a/pillar/cli/elastic.py
+++ b/pillar/cli/elastic.py
@@ -76,21 +76,26 @@ def _public_project_ids() -> typing.List[bson.ObjectId]:
 def _reindex_nodes():
     db = current_app.db()
     nodes_coll = db['nodes']
-    nodes_coll = nodes_coll.find({'project': {'$in': _public_project_ids()}})
-    node_count = nodes_coll.count()
+    nodes = nodes_coll.find({'project': {'$in': _public_project_ids()}})
+    node_count = nodes.count()
 
-    log.debug('Nodes %d will be reindexed in Elastic', node_count)
+    log.info('Nodes %d will be reindexed in Elastic', node_count)
 
     from pillar.celery.search_index_tasks import prepare_node_data
     from pillar.api.search import elastic_indexing
 
-    for node in nodes_coll:
+    indexed = 0
+    for idx, node in enumerate(nodes):
+        if idx % 100 == 0:
+            log.info('Processing node %d/%d', idx+1, node_count)
         try:
             to_index = prepare_node_data('', node=node)
             elastic_indexing.index_node_save(to_index)
         except (KeyError, AttributeError):
-            log.exception('%s is missing Field', node)
-            continue
+            log.exception('Node %s is missing Field', node)
+        else:
+            indexed += 1
+    log.info('Reindexed %d/%d nodes', indexed, node_count)
 
 
 @manager_elastic.option('indexname', nargs='?')