Improved support for files

We updated the way files are stored in the files collection. Any
derived variation of a file (different encoding or size) is stored as
new record, referencing the original as a parent.
We also added a generate_link method, which is in charge of providing
the client API with the actual link to the backend specified by the
file.

pillar/application/__init__.py

@@ -14,6 +14,7 @@ from bson import ObjectId
 
 from flask import g
 from flask import request
+from flask import url_for
 
 from pre_hooks import pre_GET
 from pre_hooks import pre_PUT
@@ -289,6 +290,27 @@ def post_GET_user(request, payload):
 
 app.on_post_GET_users += post_GET_user
 
+# Hook to check the backend of a file resource, to build an appropriate link
+# that can be used by the client to retrieve the actual file.
+def generate_link(backend, path):
+    if backend == 'pillar':
+        link = url_for('file_server.index', file_name=path, _external=True)
+    elif backend == 'cdnsun':
+        pass
+    else:
+        link = None
+    return link
+
+def before_returning_file(response):
+    response['link'] = generate_link(response['backend'], response['path'])
+
+def before_returning_files(response):
+    for item in response['_items']:
+        item['link'] = generate_link(item['backend'], item['path'])
+
+
+app.on_fetched_item_files += before_returning_file
+app.on_fetched_resource_files += before_returning_files
 
 # The file_server module needs app to be defined
 from file_server import file_server

pillar/manage.py

@@ -5,6 +5,8 @@ from flask.ext.script import Manager
 
 manager = Manager(app)
 
+MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost')
+
 @manager.command
 def runserver():
     try:
@@ -37,7 +39,7 @@ def clear_db():
     """
     from pymongo import MongoClient
 
-    client = MongoClient()
+    client = MongoClient(MONGO_HOST, 27017)
     db = client.eve
     db.drop_collection('nodes')
     db.drop_collection('node_types')
@@ -50,7 +52,7 @@ def remove_properties_order():
     """Removes properties.order
     """
     from pymongo import MongoClient
-    client = MongoClient()
+    client = MongoClient(MONGO_HOST, 27017)
     db = client.eve
     nodes = db.nodes.find()
     for node in nodes:
@@ -71,7 +73,7 @@ def upgrade_node_types():
     """
     from pymongo import MongoClient
 
-    client = MongoClient()
+    client = MongoClient(MONGO_HOST, 27017)
     db = client.eve
     node_types = db.node_types.find({})
     old_ids = {}
@@ -84,7 +86,7 @@ def get_id(collection, name):
     """Returns the _id of the given collection
     and name."""
     from pymongo import MongoClient
-    client = MongoClient()
+    client = MongoClient(MONGO_HOST, 27017)
     db = client.eve
     node = db[collection].find({'name': name})
     print (node[0]['_id'])
@@ -97,7 +99,7 @@ def manage_groups():
     and add or remove the user from that group.
     """
     from pymongo import MongoClient
-    client = MongoClient()
+    client = MongoClient(MONGO_HOST, 27017)
     db = client.eve
 
     print ("")
@@ -562,6 +564,7 @@ def populate_node_types(old_ids={}):
     group_node_type = {
         'name': 'group',
         'description': 'Generic group node type',
+        'parent': {},
         'dyn_schema': {
             'url': {
                 'type': 'string',
@@ -577,7 +580,6 @@ def populate_node_types(old_ids={}):
                 'type': 'string',
                 'maxlength': 256,
             },
-            'parent': {}
         },
         'form_schema': {
             'url': {},
@@ -586,9 +588,47 @@ def populate_node_types(old_ids={}):
         },
     }
 
+    asset_node_type = {
+        'name': 'asset',
+        'description': 'Assets for Elephants Dream',
+        # This data type does not have parent limitations (can be child
+        # of any node). An empty parent declaration is required.
+        'parent': {},
+        'dyn_schema': {
+            'status': {
+                'type': 'string',
+                'allowed': [
+                    'published',
+                    'pending',
+                    'processing'
+                ],
+            },
+            # We expose the type of asset we point to. Usually image, video,
+            # zipfile, ect.
+            'contentType':{
+                'type': 'string'
+            },
+            # We point to the original file (and use it to extract any relevant
+            # variation useful for our scope).
+            'file': {
+                'type': 'objectid',
+                'data_relation': {
+                    'resource': 'files',
+                    'field': '_id',
+                    'embeddable': True
+                },
+            }
+        },
+        'form_schema': {
+            'status': {},
+            'contentType': {},
+            'file': {},
+        }
+    }
+
     from pymongo import MongoClient
 
-    client = MongoClient()
+    client = MongoClient(MONGO_HOST, 27017)
     db = client.eve
 
     def mix_node_type(old_id, node_type_dict):
@@ -619,43 +659,60 @@ def populate_node_types(old_ids={}):
     # upgrade(scene_node_type, old_ids)
     # upgrade(act_node_type, old_ids)
     # upgrade(comment_node_type, old_ids)
-    upgrade(project_node_type, old_ids)
+    # upgrade(project_node_type, old_ids)
+    upgrade(asset_node_type, old_ids)
+
+@manager.command
+def add_group():
+    owners_group = {
+        'name': 'owners',
+        'permissions': [
+            {'node_type': get_id('node_types', 'shot'),
+             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
+             },
+            {'node_type': get_id('node_types', 'task'),
+             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
+             },
+            {'node_type': get_id('node_types', 'scene'),
+             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
+             },
+            {'node_type': get_id('node_types', 'act'),
+             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
+             },
+            {'node_type': get_id('node_types', 'comment'),
+             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
+             },
+        ]
+    }
+    post_item('groups', world_group)
 
 
 @manager.command
-def migrate_custom():
+def add_file():
-    from pymongo import MongoClient
+    from datetime import datetime
-
+    RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
-    client = MongoClient()
+    video = {
-    db = client.eve
+        'name': 'Video test',
-
+        'description': 'Video test description',
-    group_node_type = {
+        # 'parent': 'objectid',
-        'name': 'group',
+        'contentType': 'video/mp4',
-        'description': 'Generic group node type',
+        # Duration in seconds, only if it's a video
-        'dyn_schema': {
+        'duration': 50,
-            'url': {
+        'size': '720p',
-                'type': 'string',
+        'format': 'mp4',
-            },
+        'width': 1280,
-            'status': {
+        'height': 720,
-                'type': 'string',
+        'user': '552b066b41acdf5dec4436f2',
-                'allowed': [
+        'length': 15000,
-                    'published',
+        'uploadDate': datetime.strftime(
-                    'pending'
+            datetime.now(), RFC1123_DATE_FORMAT),
-                ],
+        'md5': 'md5',
-            },
+        'filename': 'The file name',
-            'notes': {
+        'backend': 'pillar',
-                'type': 'string',
+        'path': '0000.mp4',
-                'maxlength': 256,
-            },
-            'parent': {}
-        },
-        'form_schema': {
-            'url': {},
-            'status': {},
-            'notes': {},
-        },
     }
-    db.node_types.insert(group_node_type)
+    r = post_item('files', video)
+    print r
 
 
 if __name__ == '__main__':

pillar/settings.py

@@ -170,6 +170,9 @@ nodes_schema = {
         'type': 'integer',
         'minlength': 0,
     },
+    'revision': {
+        'type': 'integer',
+    },
     'parent': {
         'type': 'objectid',
          'data_relation': {
@@ -252,32 +255,45 @@ files_schema = {
         'type': 'string',
         'required': True,
     },
-    # Preview parameters:
+    # If the object has a parent, it is a variation of its parent. When querying
-    'is_preview': {
+    # for a file we are going to check if the object does NOT have a parent. In
-        'type': 'boolean'
+    # this case we will query for all files with the ObjectID as parent and we
+    # will aggregate them according of the type (if it's an image we will use
+    # some prefix, if it's a video we will combine the contentType and a custom
+    # prefix, such as 720p)
+    'parent': {
+        'type': 'objectid',
+         'data_relation': {
+            'resource': 'files',
+            'field': '_id',
+            'embeddable': True
+         },
     },
-    'size': {
+    'contentType': { # MIME type image/png video/mp4
+        'type': 'string',
+        'required': True,
+    },
+    # Duration in seconds, only if it's a video
+    'duration': {
+        'type': 'integer',
+    },
+    'size': { # xs, s, b, 720p, 2K
         'type': 'string'
     },
-    'format': {
+    'format': { # human readable format, like mp4, HLS, webm, mov
         'type': 'string'
     },
-    'width': {
+    'width': { # valid for images and video contentType
         'type': 'integer'
     },
     'height': {
         'type': 'integer'
     },
-    #
     'user': {
         'type': 'objectid',
         'required': True,
     },
-    'contentType': {
+    'length': { # Size in bytes
-        'type': 'string',
-        'required': True,
-    },
-    'length': {
         'type': 'integer',
         'required': True,
     },
@@ -296,14 +312,14 @@ files_schema = {
     'backend': {
         'type': 'string',
         'required': True,
-        'allowed': ["attract-web", "attract"]
+        'allowed': ["attract-web", "pillar"]
     },
     'path': {
         'type': 'string',
         'required': True,
         'unique': True,
     },
-    'previews': {
+    'previews': { # Deprecated (see comments above)
         'type': 'list',
         'schema': {
             'type': 'objectid',
@@ -313,6 +329,10 @@ files_schema = {
                 'embeddable': True
             }
         }
+    },
+    # Preview parameters:
+    'is_preview': { # Deprecated
+        'type': 'boolean'
     }
 }