From 7cecfbe4e0cbfcf96c5ced93a78f608da606579f Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Tue, 8 Sep 2015 15:06:45 +0200 Subject: [PATCH] Improved support for files We updated the way files are stored in the files collection. Any derived variation of a file (different encoding or size) is stored as new record, referencing the original as a parent. We also added a generate_link method, which is in charge of providing the client API with the actual link to the backend specified by the file. --- pillar/application/__init__.py | 22 ++++++ pillar/manage.py | 137 +++++++++++++++++++++++---------- pillar/settings.py | 48 ++++++++---- 3 files changed, 153 insertions(+), 54 deletions(-) diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 3e4f55eb..cd2b569f 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -14,6 +14,7 @@ from bson import ObjectId from flask import g from flask import request +from flask import url_for from pre_hooks import pre_GET from pre_hooks import pre_PUT @@ -289,6 +290,27 @@ def post_GET_user(request, payload): app.on_post_GET_users += post_GET_user +# Hook to check the backend of a file resource, to build an appropriate link +# that can be used by the client to retrieve the actual file. +def generate_link(backend, path): + if backend == 'pillar': + link = url_for('file_server.index', file_name=path, _external=True) + elif backend == 'cdnsun': + pass + else: + link = None + return link + +def before_returning_file(response): + response['link'] = generate_link(response['backend'], response['path']) + +def before_returning_files(response): + for item in response['_items']: + item['link'] = generate_link(item['backend'], item['path']) + + +app.on_fetched_item_files += before_returning_file +app.on_fetched_resource_files += before_returning_files # The file_server module needs app to be defined from file_server import file_server diff --git a/pillar/manage.py b/pillar/manage.py index 1c127dae..11f69cf6 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -5,6 +5,8 @@ from flask.ext.script import Manager manager = Manager(app) +MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost') + @manager.command def runserver(): try: @@ -37,7 +39,7 @@ def clear_db(): """ from pymongo import MongoClient - client = MongoClient() + client = MongoClient(MONGO_HOST, 27017) db = client.eve db.drop_collection('nodes') db.drop_collection('node_types') @@ -50,7 +52,7 @@ def remove_properties_order(): """Removes properties.order """ from pymongo import MongoClient - client = MongoClient() + client = MongoClient(MONGO_HOST, 27017) db = client.eve nodes = db.nodes.find() for node in nodes: @@ -71,7 +73,7 @@ def upgrade_node_types(): """ from pymongo import MongoClient - client = MongoClient() + client = MongoClient(MONGO_HOST, 27017) db = client.eve node_types = db.node_types.find({}) old_ids = {} @@ -84,7 +86,7 @@ def get_id(collection, name): """Returns the _id of the given collection and name.""" from pymongo import MongoClient - client = MongoClient() + client = MongoClient(MONGO_HOST, 27017) db = client.eve node = db[collection].find({'name': name}) print (node[0]['_id']) @@ -97,7 +99,7 @@ def manage_groups(): and add or remove the user from that group. """ from pymongo import MongoClient - client = MongoClient() + client = MongoClient(MONGO_HOST, 27017) db = client.eve print ("") @@ -562,6 +564,7 @@ def populate_node_types(old_ids={}): group_node_type = { 'name': 'group', 'description': 'Generic group node type', + 'parent': {}, 'dyn_schema': { 'url': { 'type': 'string', @@ -577,7 +580,6 @@ def populate_node_types(old_ids={}): 'type': 'string', 'maxlength': 256, }, - 'parent': {} }, 'form_schema': { 'url': {}, @@ -586,9 +588,47 @@ def populate_node_types(old_ids={}): }, } + asset_node_type = { + 'name': 'asset', + 'description': 'Assets for Elephants Dream', + # This data type does not have parent limitations (can be child + # of any node). An empty parent declaration is required. + 'parent': {}, + 'dyn_schema': { + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'pending', + 'processing' + ], + }, + # We expose the type of asset we point to. Usually image, video, + # zipfile, ect. + 'contentType':{ + 'type': 'string' + }, + # We point to the original file (and use it to extract any relevant + # variation useful for our scope). + 'file': { + 'type': 'objectid', + 'data_relation': { + 'resource': 'files', + 'field': '_id', + 'embeddable': True + }, + } + }, + 'form_schema': { + 'status': {}, + 'contentType': {}, + 'file': {}, + } + } + from pymongo import MongoClient - client = MongoClient() + client = MongoClient(MONGO_HOST, 27017) db = client.eve def mix_node_type(old_id, node_type_dict): @@ -619,43 +659,60 @@ def populate_node_types(old_ids={}): # upgrade(scene_node_type, old_ids) # upgrade(act_node_type, old_ids) # upgrade(comment_node_type, old_ids) - upgrade(project_node_type, old_ids) + # upgrade(project_node_type, old_ids) + upgrade(asset_node_type, old_ids) + +@manager.command +def add_group(): + owners_group = { + 'name': 'owners', + 'permissions': [ + {'node_type': get_id('node_types', 'shot'), + 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] + }, + {'node_type': get_id('node_types', 'task'), + 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] + }, + {'node_type': get_id('node_types', 'scene'), + 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] + }, + {'node_type': get_id('node_types', 'act'), + 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] + }, + {'node_type': get_id('node_types', 'comment'), + 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] + }, + ] + } + post_item('groups', world_group) @manager.command -def migrate_custom(): - from pymongo import MongoClient - - client = MongoClient() - db = client.eve - - group_node_type = { - 'name': 'group', - 'description': 'Generic group node type', - 'dyn_schema': { - 'url': { - 'type': 'string', - }, - 'status': { - 'type': 'string', - 'allowed': [ - 'published', - 'pending' - ], - }, - 'notes': { - 'type': 'string', - 'maxlength': 256, - }, - 'parent': {} - }, - 'form_schema': { - 'url': {}, - 'status': {}, - 'notes': {}, - }, +def add_file(): + from datetime import datetime + RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' + video = { + 'name': 'Video test', + 'description': 'Video test description', + # 'parent': 'objectid', + 'contentType': 'video/mp4', + # Duration in seconds, only if it's a video + 'duration': 50, + 'size': '720p', + 'format': 'mp4', + 'width': 1280, + 'height': 720, + 'user': '552b066b41acdf5dec4436f2', + 'length': 15000, + 'uploadDate': datetime.strftime( + datetime.now(), RFC1123_DATE_FORMAT), + 'md5': 'md5', + 'filename': 'The file name', + 'backend': 'pillar', + 'path': '0000.mp4', } - db.node_types.insert(group_node_type) + r = post_item('files', video) + print r if __name__ == '__main__': diff --git a/pillar/settings.py b/pillar/settings.py index 4b805fe0..3800bfbf 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -170,6 +170,9 @@ nodes_schema = { 'type': 'integer', 'minlength': 0, }, + 'revision': { + 'type': 'integer', + }, 'parent': { 'type': 'objectid', 'data_relation': { @@ -252,32 +255,45 @@ files_schema = { 'type': 'string', 'required': True, }, - # Preview parameters: - 'is_preview': { - 'type': 'boolean' + # If the object has a parent, it is a variation of its parent. When querying + # for a file we are going to check if the object does NOT have a parent. In + # this case we will query for all files with the ObjectID as parent and we + # will aggregate them according of the type (if it's an image we will use + # some prefix, if it's a video we will combine the contentType and a custom + # prefix, such as 720p) + 'parent': { + 'type': 'objectid', + 'data_relation': { + 'resource': 'files', + 'field': '_id', + 'embeddable': True + }, }, - 'size': { + 'contentType': { # MIME type image/png video/mp4 + 'type': 'string', + 'required': True, + }, + # Duration in seconds, only if it's a video + 'duration': { + 'type': 'integer', + }, + 'size': { # xs, s, b, 720p, 2K 'type': 'string' }, - 'format': { + 'format': { # human readable format, like mp4, HLS, webm, mov 'type': 'string' }, - 'width': { + 'width': { # valid for images and video contentType 'type': 'integer' }, 'height': { 'type': 'integer' }, - # 'user': { 'type': 'objectid', 'required': True, }, - 'contentType': { - 'type': 'string', - 'required': True, - }, - 'length': { + 'length': { # Size in bytes 'type': 'integer', 'required': True, }, @@ -296,14 +312,14 @@ files_schema = { 'backend': { 'type': 'string', 'required': True, - 'allowed': ["attract-web", "attract"] + 'allowed': ["attract-web", "pillar"] }, 'path': { 'type': 'string', 'required': True, 'unique': True, }, - 'previews': { + 'previews': { # Deprecated (see comments above) 'type': 'list', 'schema': { 'type': 'objectid', @@ -313,6 +329,10 @@ files_schema = { 'embeddable': True } } + }, + # Preview parameters: + 'is_preview': { # Deprecated + 'type': 'boolean' } }