archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

refresh_links_for_backend: handle more malformed cases and exceptions

Browse Source
This commit is contained in:
Sybren A. Stüvel 2016-07-27 11:06:02 +02:00
parent 943697fcd2
commit de36f1727a
1 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pillar/application/modules/file_storage.py
View File

@ -452,7 +452,7 @@ def refresh_links_for_project(project_uuid, chunk_size, expiry_seconds):
def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds):
from flask import current_app import gcloud.exceptions
# Retrieve expired links. # Retrieve expired links.
files_collection = current_app.data.driver.db['files'] files_collection = current_app.data.driver.db['files']
@ -474,18 +474,30 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds):
refreshed = 0 refreshed = 0
for file_doc in to_refresh: for file_doc in to_refresh:
file_id = file_doc['_id']
project_id = file_doc.get('project') project_id = file_doc.get('project')
if project_id is None: if project_id is None:
log.debug('Skipping file %s, it has no project.', file_doc['_id']) log.debug('Skipping file %s, it has no project.', file_id)
continue continue
count = proj_coll.count({'_id': project_id}) count = proj_coll.count({'_id': project_id})
if count == 0: if count == 0:
log.debug('Skipping file %s, project does not exist.', file_doc['_id']) log.debug('Skipping file %s, project does not exist.', file_id)
continue continue
log.debug('Refreshing links for file %s', file_doc['_id']) if 'file_path' not in file_doc:
log.warning("File %s malformed: missing 'file_path' property.", file_id)
continue
log.debug('Refreshing links for file %s', file_id)
try:
_generate_all_links(file_doc, now) _generate_all_links(file_doc, now)
except gcloud.exceptions.Forbidden:
log.warning('GCS forbids us access to project %s bucket for file %s.',
project_id, file_id)
continue
refreshed += 1 refreshed += 1
log.info('Refreshed %i links', refreshed) log.info('Refreshed %i links', refreshed)