From eba28b4eb4df86e9a7762496c58e95374fb0dbf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Thu, 14 Sep 2017 15:10:09 +0200 Subject: [PATCH] File link refresh: report on every N refreshed links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it easier to see what the Celery worker is actually working on when refreshing a large number of links. It'll report on every N refreshed links, where N = link_count/25 but clamped to N ∈ [5, 100] --- pillar/api/file_storage/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pillar/api/file_storage/__init__.py b/pillar/api/file_storage/__init__.py index 8d9fb824..4e98cdc9 100644 --- a/pillar/api/file_storage/__init__.py +++ b/pillar/api/file_storage/__init__.py @@ -555,6 +555,7 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): log.info('Found %d documents to refresh.', document_count) refreshed = 0 + report_chunks = min(max(5, document_count // 25), 100) for file_doc in to_refresh: try: file_id = file_doc['_id'] @@ -587,6 +588,9 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): 'project %s bucket.', file_id, project_id) continue refreshed += 1 + + if refreshed % report_chunks == 0: + log.info('Refreshed %i links', refreshed) except KeyboardInterrupt: log.warning('Aborting due to KeyboardInterrupt after refreshing %i ' 'links', refreshed)