diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 1eff40e8..4052eddd 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -102,7 +102,7 @@ logging.getLogger('werkzeug').setLevel(logging.INFO) log = logging.getLogger(__name__) log.setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO) -log.info('Pillar starting') +log.info('Pillar starting, debug=%s', app.config['DEBUG']) bugsnag.configure( api_key=app.config['BUGSNAG_API_KEY'], @@ -148,9 +148,7 @@ from utils.algolia import algolia_index_node_save from utils.activities import activity_subscribe from utils.activities import activity_object_add from utils.activities import notification_parse -from modules.file_storage import process_file -from modules.file_storage import delete_file -from modules.file_storage import generate_link +from .modules import file_storage def before_returning_item_permissions(response): @@ -261,7 +259,7 @@ def item_parse_attachments(response): # Get the correc variation from the file thumbnail = next((item for item in f['variations'] if item['size'] == size), None) - l = generate_link(f['backend'], thumbnail['file_path'], str(f['project'])) + l = file_storage.generate_link(f['backend'], thumbnail['file_path'], str(f['project'])) # Build Markdown img string l = '![{0}]({1} "{2}")'.format(slug, l, f['name']) # Parse the content of the file and replace the attachment @@ -351,36 +349,17 @@ def post_POST_files(request, payload): """After an file object has been created, we do the necessary processing and further update it. """ - process_file(request.get_json()) + file_storage.process_file(request.get_json()) app.on_post_POST_files += post_POST_files - -def before_returning_file(response): - # TODO: add project id to all files - project_id = None if 'project' not in response else str(response['project']) - response['link'] = generate_link( - response['backend'], response['file_path'], project_id) - if 'variations' in response: - for variation in response['variations']: - variation['link'] = generate_link( - response['backend'], variation['file_path'], project_id) - - -def before_returning_files(response): - for item in response['_items']: - # TODO: add project id to all files - project_id = None if 'project' not in item else str(item['project']) - item['link'] = generate_link(item['backend'], item['file_path'], project_id) - - -app.on_fetched_item_files += before_returning_file -app.on_fetched_resource_files += before_returning_files +app.on_fetched_item_files += file_storage.before_returning_file +app.on_fetched_resource_files += file_storage.before_returning_files def before_deleting_file(item): - delete_file(item) + file_storage.delete_file(item) app.on_delete_item_files += before_deleting_file diff --git a/pillar/application/modules/file_storage.py b/pillar/application/modules/file_storage.py index 47053a67..77795aa6 100644 --- a/pillar/application/modules/file_storage.py +++ b/pillar/application/modules/file_storage.py @@ -1,6 +1,9 @@ +import datetime import logging import os from multiprocessing import Process + +import bson.tz_util from bson import ObjectId from flask import request from flask import Blueprint @@ -8,7 +11,10 @@ from flask import jsonify from flask import send_from_directory from flask import url_for, helpers from eve.methods.put import put_internal +from eve.methods.patch import patch_internal + from application import app +from application.utils import remove_private_keys from application.utils.imaging import generate_local_thumbnails from application.utils.imaging import get_video_data from application.utils.imaging import ffmpeg_encode @@ -298,3 +304,52 @@ def generate_link(backend, file_path, project_id=None, is_public=False): else: link = None return link + + +def before_returning_file(response): + ensure_valid_link(response) + + +def before_returning_files(response): + for item in response['_items']: + ensure_valid_link(item) + + +def ensure_valid_link(response): + """Ensures the file item has valid file links using generate_link(...).""" + + log.debug('Inspecting link for file %s', response['_id']) + + # Check link expiry. + now = datetime.datetime.now(tz=bson.tz_util.utc) + if 'link_expires' in response: + link_expires = response['link_expires'] + if now < link_expires: + # Not expired yet, so don't bother regenerating anything. + log.debug('Link expires at %s, which is in the future, so not generating new link', link_expires) + return + + log.debug('Link expired at %s, which is in the past; generating new link', link_expires) + else: + log.debug('No expiry date for link; generating new link') + + # Generate a new link for the file and all its variations. + project_id = str(response['project']) if 'project' in response else None # TODO: add project id to all files + backend = response['backend'] + response['link'] = generate_link(backend, response['file_path'], project_id) + if 'variations' in response: + for variation in response['variations']: + variation['link'] = generate_link(backend, variation['file_path'], project_id) + + # Construct the new expiry datetime. + validity_secs = app.config['FILE_LINK_VALIDITY'][backend] + response['link_expires'] = now + datetime.timedelta(seconds=validity_secs) + + patch_info = remove_private_keys(response) + (patch_resp, _, _, _) = patch_internal('files', patch_info, _id=ObjectId(response['_id'])) + if patch_resp.get('_status') == 'ERR': + log.warning('Unable to save new links for file %s: %r', response['_id'], patch_resp) + # TODO: raise a snag. + response['_updated'] = now + else: + response['_updated'] = patch_resp['_updated'] diff --git a/pillar/application/utils/__init__.py b/pillar/application/utils/__init__.py index e69de29b..76e970b8 100644 --- a/pillar/application/utils/__init__.py +++ b/pillar/application/utils/__init__.py @@ -0,0 +1,12 @@ +import copy + + +def remove_private_keys(document): + """Removes any key that starts with an underscore, returns result as new dictionary.""" + + patch_info = copy.deepcopy(document) + for key in list(patch_info.keys()): + if key.startswith('_'): + del patch_info[key] + + return patch_info diff --git a/pillar/config.py.example b/pillar/config.py.example index de043517..68301e72 100644 --- a/pillar/config.py.example +++ b/pillar/config.py.example @@ -65,5 +65,12 @@ class Development(object): ALGOLIA_API_KEY = '' ALGOLIA_INDEX_USERS = '' + # Validity period of links, per file storage backend. Expressed in seconds. + # Shouldn't be more than a year, as this isn't supported by HTTP/1.1. + FILE_LINK_VALIDITY = defaultdict( + lambda: 3600 * 24 * 30, # default of 1 month. + gcs=3600 * 23, # 23 hours for Google Cloud Storage. + ) + class Deployment(Development): pass diff --git a/pillar/settings.py b/pillar/settings.py index 62f4c136..11ebea32 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -413,6 +413,12 @@ files_schema = { #'required': True, 'unique': True, }, + 'link': { + 'type': 'string', + }, + 'link_expires': { + 'type': 'datetime', + }, 'project': { # The project node the files belongs to (does not matter if it is # attached to an asset or something else). We use the project id as @@ -463,6 +469,9 @@ files_schema = { 'file_path': { 'type': 'string', }, + 'link': { + 'type': 'string', + } } } },