archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

refresh_links_for_backend: handling KeyboardInterrupt better

Browse Source 
It now reports on the number of refreshed links before aborting.
This commit is contained in:
Sybren A. Stüvel 2016-07-27 11:58:39 +02:00
parent fbf7dd9596
commit 1312ededfb
2 changed files with 32 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pillar/application/modules/file_storage.py
View File

@ -474,35 +474,39 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds):
refreshed = 0
for file_doc in to_refresh:
file_id = file_doc['_id']
project_id = file_doc.get('project')
if project_id is None:
log.debug('Skipping file %s, it has no project.', file_id)
continue
count = proj_coll.count({'_id': project_id, '$or': [
{'_deleted': {'$exists': False}},
{'_deleted': False},
]})
if count == 0:
log.debug('Skipping file %s, project %s does not exist.', file_id, project_id)
continue
if 'file_path' not in file_doc:
log.warning("Skipping file %s, missing 'file_path' property.", file_id)
continue
log.debug('Refreshing links for file %s', file_id)
try:
_generate_all_links(file_doc, now)
except gcloud.exceptions.Forbidden:
log.warning('Skipping file %s, GCS forbids us access to project %s bucket.',
file_id, project_id)
continue
file_id = file_doc['_id']
project_id = file_doc.get('project')
if project_id is None:
log.debug('Skipping file %s, it has no project.', file_id)
continue
refreshed += 1
count = proj_coll.count({'_id': project_id, '$or': [
{'_deleted': {'$exists': False}},
{'_deleted': False},
]})
if count == 0:
log.debug('Skipping file %s, project %s does not exist.', file_id, project_id)
continue
if 'file_path' not in file_doc:
log.warning("Skipping file %s, missing 'file_path' property.", file_id)
continue
log.debug('Refreshing links for file %s', file_id)
try:
_generate_all_links(file_doc, now)
except gcloud.exceptions.Forbidden:
log.warning('Skipping file %s, GCS forbids us access to project %s bucket.',
file_id, project_id)
continue
refreshed += 1
except KeyboardInterrupt:
log.warning('Aborting due to KeyboardInterrupt after refreshing %i links',
refreshed)
return
log.info('Refreshed %i links', refreshed)

5
pillar/manage.py
View File

@ -593,10 +593,7 @@ def refresh_backend_links(backend_name, chunk_size=50, quiet=False, window=12):
chunk_size = int(chunk_size) # CLI parameters are passed as strings
from application.modules import file_storage
try:
file_storage.refresh_links_for_backend(backend_name, chunk_size, window * 3600)
except KeyboardInterrupt:
print('Aborted due to keyboard interrupt.')
file_storage.refresh_links_for_backend(backend_name, chunk_size, window * 3600)
@manager.command