From beac125ff9c2d1e8e308d7de4134b9e3bd0a2f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Thu, 20 Dec 2018 12:51:53 +0100 Subject: [PATCH] Nicer logging when refreshing file links --- pillar/api/file_storage/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pillar/api/file_storage/__init__.py b/pillar/api/file_storage/__init__.py index 5f6445b2..2c686eb2 100644 --- a/pillar/api/file_storage/__init__.py +++ b/pillar/api/file_storage/__init__.py @@ -609,6 +609,7 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): import gcloud.exceptions my_log = log.getChild(f'refresh_links_for_backend.{backend_name}') + start_time = time.time() # Retrieve expired links. files_collection = current_app.data.driver.db['files'] @@ -632,10 +633,10 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): return if 0 < chunk_size == document_count: - my_log.info('Found %d documents to refresh, probably limited by the chunk size.', - document_count) + my_log.info('Found %d documents to refresh, probably limited by the chunk size %d', + document_count, chunk_size) else: - my_log.info('Found %d documents to refresh.', document_count) + my_log.info('Found %d documents to refresh, chunk size=%d', document_count, chunk_size) refreshed = 0 report_chunks = min(max(5, document_count // 25), 100) @@ -679,8 +680,10 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): 'links', refreshed) return - my_log.info('Refreshed %i links', refreshed) + if refreshed % report_chunks != 0: + my_log.info('Refreshed %i links', refreshed) + my_log.info('Refresh took %s', datetime.timedelta(seconds=time.time() - start)) @require_login() def create_file_doc(name, filename, content_type, length, project,