diff --git a/pillar/application/modules/file_storage.py b/pillar/application/modules/file_storage.py index 76dc4de8..07ebe648 100644 --- a/pillar/application/modules/file_storage.py +++ b/pillar/application/modules/file_storage.py @@ -8,12 +8,14 @@ import eve.utils from bson import ObjectId from eve.methods.patch import patch_internal from eve.methods.put import put_internal -from flask import Blueprint +from flask import Blueprint, safe_join from flask import jsonify from flask import request +from flask import abort from flask import send_from_directory from flask import url_for, helpers +from application import utils from application.utils import remove_private_keys from application.utils.cdn import hash_file_path from application.utils.encoding import Encoder @@ -75,8 +77,8 @@ def build_thumbnails(file_path=None, file_id=None): file_ = files_collection.find_one({"_id": ObjectId(file_id)}) file_path = file_['name'] - file_full_path = os.path.join(app.config['SHARED_DIR'], file_path[:2], - file_path) + file_full_path = safe_join(safe_join(app.config['SHARED_DIR'], file_path[:2]), + file_path) # Does the original file exist? if not os.path.isfile(file_full_path): return "", 404 @@ -147,26 +149,29 @@ def index(file_name=None): return jsonify({'url': url_for('file_storage.index', file_name=file_name)}) -def process_file(src_file): - """Process the file +def process_file(file_id, src_file): + """Process the file. + + :param file_id: '_id' key of the file + :param src_file: POSTed data of the file, lacks private properties. """ from application import app - file_id = src_file['_id'] - # Remove properties that do not belong in the collection - internal_fields = ['_id', '_etag', '_updated', '_created', '_status'] - for field in internal_fields: - src_file.pop(field, None) + src_file = utils.remove_private_keys(src_file) - files_collection = app.data.driver.db['files'] - file_abs_path = os.path.join( - app.config['SHARED_DIR'], src_file['name'][:2], src_file['name']) + filename = src_file['name'] + file_abs_path = safe_join(safe_join(app.config['SHARED_DIR'], filename[:2]), filename) + + if not os.path.exists(file_abs_path): + log.warning("POSTed file document %r refers to non-existant file on file system %s!", + file_id, file_abs_path) + abort(422, "POSTed file document refers to non-existant file on file system!") src_file['length'] = os.stat(file_abs_path).st_size content_type = src_file['content_type'].split('/') src_file['format'] = content_type[1] mime_type = content_type[0] - src_file['file_path'] = src_file['name'] + src_file['file_path'] = filename if mime_type == 'image': from PIL import Image @@ -197,7 +202,7 @@ def process_file(src_file): src_file['variations'] = [] # Create variations for v in variations: - root, ext = os.path.splitext(src_file['name']) + root, ext = os.path.splitext(filename) filename = "{0}-{1}p.{2}".format(root, res_y, v) video_duration = None if src_video_data['duration']: @@ -250,6 +255,9 @@ def process_file(src_file): p = Process(target=encode, args=(file_abs_path, src_file, res_y)) p.start() + else: + log.info("POSTed file was of type %r, which isn't thumbnailed/encoded.", mime_type) + if mime_type != 'video': # Sync the whole subdir sync_path = os.path.split(file_abs_path)[0] @@ -259,7 +267,7 @@ def process_file(src_file): p.start() # Update the original file with additional info, e.g. image resolution - r = put_internal('files', src_file, **{'_id': ObjectId(file_id)}) + put_internal('files', src_file, _id=ObjectId(file_id)) def delete_file(file_item): @@ -383,7 +391,14 @@ def post_POST_files(request, payload): """After an file object has been created, we do the necessary processing and further update it. """ - process_file(request.get_json()) + + if 200 <= payload.status_code < 300: + import json + posted_properties = json.loads(request.data) + private_properties = json.loads(payload.data) + file_id = private_properties['_id'] + + process_file(file_id, posted_properties) def before_deleting_file(item): diff --git a/pillar/application/utils/__init__.py b/pillar/application/utils/__init__.py index b15edcb2..c56c455e 100644 --- a/pillar/application/utils/__init__.py +++ b/pillar/application/utils/__init__.py @@ -1,4 +1,9 @@ import copy +import json +import datetime + +import bson +from eve import RFC1123_DATE_FORMAT def remove_private_keys(document): @@ -11,3 +16,19 @@ def remove_private_keys(document): del patch_info[key] return patch_info + + +class PillarJSONEncoder(json.JSONEncoder): + """JSON encoder with support for Pillar resources.""" + + def default(self, obj): + if isinstance(obj, datetime.datetime): + if obj.tzinfo is None: + raise ValueError('All datetime.datetime objects should be timezone-aware.') + return obj.strftime(RFC1123_DATE_FORMAT) + + if isinstance(obj, bson.ObjectId): + return str(obj) + + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) diff --git a/pillar/application/utils/storage.py b/pillar/application/utils/storage.py index 854f086c..218729ef 100644 --- a/pillar/application/utils/storage.py +++ b/pillar/application/utils/storage.py @@ -70,7 +70,7 @@ def push_to_storage(project_id, full_path, backend='cgs'): # XXX Make public on the fly if it's an image and small preview. # This should happen by reading the database (push to storage # should change to accomodate it). - if full_path.endswith('-t.jpg'): + if blob is not None and full_path.endswith('-t.jpg'): blob.make_public() os.remove(full_path) diff --git a/tests/BlenderDesktopLogo.png b/tests/BlenderDesktopLogo.png new file mode 100644 index 00000000..fd976d7a Binary files /dev/null and b/tests/BlenderDesktopLogo.png differ diff --git a/tests/common_test_class.py b/tests/common_test_class.py index 45263ec8..b52f35d7 100644 --- a/tests/common_test_class.py +++ b/tests/common_test_class.py @@ -34,6 +34,7 @@ class AbstractPillarTest(TestMinimal): app.config['BLENDER_ID_ENDPOINT'] = BLENDER_ID_ENDPOINT logging.getLogger('application').setLevel(logging.DEBUG) logging.getLogger('werkzeug').setLevel(logging.DEBUG) + logging.getLogger('eve').setLevel(logging.DEBUG) self.app = app self.client = app.test_client() @@ -59,7 +60,20 @@ class AbstractPillarTest(TestMinimal): projects_collection.insert_one(EXAMPLE_PROJECT) result = files_collection.insert_one(file) file_id = result.inserted_id - return file_id, EXAMPLE_FILE + return file_id, file + + def ensure_project_exists(self, project_overrides=None): + with self.app.test_request_context(): + projects_collection = self.app.data.driver.db['projects'] + assert isinstance(projects_collection, pymongo.collection.Collection) + + project = copy.deepcopy(EXAMPLE_PROJECT) + if project_overrides is not None: + project.update(project_overrides) + + result = projects_collection.insert_one(project) + project_id = result.inserted_id + return project_id, project def htp_blenderid_validate_unhappy(self): """Sets up HTTPretty to mock unhappy validation flow.""" diff --git a/tests/test_file_storage.py b/tests/test_file_storage.py new file mode 100644 index 00000000..b0c64b3d --- /dev/null +++ b/tests/test_file_storage.py @@ -0,0 +1,58 @@ +"""Test cases for file handling.""" + +from __future__ import print_function + +import os +import shutil +import copy +import json + +from common_test_class import AbstractPillarTest, MY_PATH +from common_test_data import EXAMPLE_FILE + + +class FileUploadingTest(AbstractPillarTest): + + def test_create_file_missing_on_fs(self): + from application import utils + from application.utils import PillarJSONEncoder + + to_post = utils.remove_private_keys(EXAMPLE_FILE) + json_file = json.dumps(to_post, cls=PillarJSONEncoder) + + with self.app.test_request_context(): + self.ensure_project_exists() + + resp = self.client.post('/files', + data=json_file, + headers={'Content-Type': 'application/json'}) + + self.assertEqual(422, resp.status_code) + + + def test_create_file_exists_on_fs(self): + from application import utils + from application.utils import PillarJSONEncoder + + filename = 'BlenderDesktopLogo.png' + full_file = copy.deepcopy(EXAMPLE_FILE) + full_file[u'name'] = filename + to_post = utils.remove_private_keys(full_file) + json_file = json.dumps(to_post, cls=PillarJSONEncoder) + + with self.app.test_request_context(): + self.ensure_project_exists() + + target_dir = os.path.join(self.app.config['SHARED_DIR'], filename[:2]) + if os.path.exists(target_dir): + assert os.path.isdir(target_dir) + else: + os.makedirs(target_dir) + shutil.copy(os.path.join(MY_PATH, filename), target_dir) + + resp = self.client.post('/files', + data=json_file, + headers={'Content-Type': 'application/json'}) + + self.assertEqual(201, resp.status_code) +