Migration script and utils for CDN link protection

pillar/application/__init__.py

@@ -224,6 +224,8 @@ def post_item(entry, data):
 
 app = Eve(validator=ValidateCustomFields, auth=CustomTokenAuth)
 
+import config
+app.config.from_object(config.Deployment)
 
 def global_validation():
     setattr(g, 'token_data', validate_token())
@@ -290,13 +292,14 @@ def post_GET_user(request, payload):
 
 app.on_post_GET_users += post_GET_user
 
+from utils import hash_file_path
 # Hook to check the backend of a file resource, to build an appropriate link
 # that can be used by the client to retrieve the actual file.
 def generate_link(backend, path):
     if backend == 'pillar':
         link = url_for('file_server.index', file_name=path, _external=True)
     elif backend == 'cdnsun':
-        pass
+        link = hash_file_path(path, None)
     else:
         link = None
     return link

pillar/application/utils.py

@@ -0,0 +1,39 @@
+import datetime
+from hashlib import md5
+from application import app
+
+def hash_file_path(file_path, expiry_timestamp=None):
+  service_domain = app.config['CDN_SERVICE_DOMAIN']
+  domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER']
+  asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \
+    '://' + \
+    service_domain + \
+    domain_subfolder + \
+    file_path
+
+  if app.config['CDN_USE_URL_SIGNING']:
+
+    url_signing_key = app.config['CDN_URL_SIGNING_KEY']
+    if not file_path.startswith('/'):
+        file_path = '/' + file_path;
+
+    hash_string = domain_subfolder + file_path + url_signing_key;
+
+    if not expiry_timestamp:
+      expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=12)
+      expiry_timestamp = expiry_timestamp.strftime('%s')
+
+    hash_string = expiry_timestamp + hash_string;
+
+    expiry_timestamp = "," + str(expiry_timestamp);
+
+    hashed_file_path = md5(hash_string).digest().encode('base64')[:-1]
+    hashed_file_path = hashed_file_path.replace('+', '-')
+    hashed_file_path = hashed_file_path.replace('/', '_')
+
+    asset_url = asset_url + \
+      '?secure=' + \
+      hashed_file_path + \
+      expiry_timestamp
+
+  return asset_url

pillar/config.py.example

@@ -1,5 +1,4 @@
 import os
-
 class Config(object):
     # Configured for GMAIL
     MAIL_SERVER = ''
@@ -9,20 +8,13 @@ class Config(object):
     MAIL_PASSWORD = ''
     DEFAULT_MAIL_SENDER = ''
 
-    # Flask-Security setup
-    SECURITY_LOGIN_WITHOUT_CONFIRMATION = True
-    SECURITY_REGISTERABLE = True
-    SECURITY_RECOVERABLE = True
-    SECURITY_CHANGEABLE = True
-    SECUIRTY_POST_LOGIN = '/'
-    SECURITY_PASSWORD_HASH = ''
-    SECURITY_PASSWORD_SALT = ''
-    SECURITY_EMAIL_SENDER = ''
-
 class Development(Config):
-    SECRET_KEY = ''
-    HOST = '0.0.0.0'
-    PORT = 5000
-    DEBUG = True
     FILE_STORAGE = '{0}/application/static/storage'.format(
         os.path.join(os.path.dirname(__file__)))
+    CDN_USE_URL_SIGNING = False
+    CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
+    CDN_SERVICE_DOMAIN = ''
+    CDN_CONTENT_SUBFOLDER = ''
+    CDN_URL_SIGNING_KEY = ''
+    CDN_STORAGE_USER = ''
+    CDN_STORAGE_ADDRESS = ''

pillar/manage.py

@@ -241,7 +241,7 @@ def add_groups():
 
 
 @manager.command
-def populate_db_test():
+def populate_db():
     """Populate the db with sample data
     """
     populate_node_types()
@@ -593,7 +593,9 @@ def populate_node_types(old_ids={}):
         'description': 'Assets for Elephants Dream',
         # This data type does not have parent limitations (can be child
         # of any node). An empty parent declaration is required.
-        'parent': {},
+        'parent': {
+            "node_types": ["group",]
+        },
         'dyn_schema': {
             'status': {
                 'type': 'string',
@@ -605,7 +607,7 @@ def populate_node_types(old_ids={}):
             },
             # We expose the type of asset we point to. Usually image, video,
             # zipfile, ect.
-            'contentType':{
+            'content_type':{
                 'type': 'string'
             },
             # We point to the original file (and use it to extract any relevant
@@ -621,7 +623,7 @@ def populate_node_types(old_ids={}):
         },
         'form_schema': {
             'status': {},
-            'contentType': {},
+            'content_type': {},
             'file': {},
         }
     }
@@ -662,33 +664,9 @@ def populate_node_types(old_ids={}):
     # upgrade(project_node_type, old_ids)
     upgrade(asset_node_type, old_ids)
 
-@manager.command
-def add_group():
-    owners_group = {
-        'name': 'owners',
-        'permissions': [
-            {'node_type': get_id('node_types', 'shot'),
-             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
-             },
-            {'node_type': get_id('node_types', 'task'),
-             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
-             },
-            {'node_type': get_id('node_types', 'scene'),
-             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
-             },
-            {'node_type': get_id('node_types', 'act'),
-             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
-             },
-            {'node_type': get_id('node_types', 'comment'),
-             'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
-             },
-        ]
-    }
-    post_item('groups', world_group)
-
 
 @manager.command
-def add_file():
+def add_file_video():
     from datetime import datetime
     RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
     video = {
@@ -704,15 +682,217 @@ def add_file():
         'height': 720,
         'user': '552b066b41acdf5dec4436f2',
         'length': 15000,
-        'uploadDate': datetime.strftime(
-            datetime.now(), RFC1123_DATE_FORMAT),
         'md5': 'md5',
         'filename': 'The file name',
         'backend': 'pillar',
         'path': '0000.mp4',
     }
     r = post_item('files', video)
-    print r
+    return r
+
+@manager.command
+def add_node_asset(file_id):
+    """Creates a node of type asset, starting from an existing file id.
+    :param file_id: the file id to use
+    :param picture_id: the picture id to use
+    :param group_id: the parent folder
+    """
+    picture_id = None
+    parent_id = None
+
+    from bson.objectid import ObjectId
+    from pymongo import MongoClient
+    client = MongoClient(MONGO_HOST, 27017)
+    db = client.eve
+    file_object = db.files.find_one({"_id": ObjectId(file_id)})
+    node_type = db.node_types.find_one({"name": "asset"})
+
+    print file_object['contentType'].split('/')[0]
+
+    node = {
+        'name': file_object['name'],
+        'description': file_object['description'],
+        #'picture': picture_id,
+        #'parent': parent_id,
+        'user': file_object['user'],
+        'node_type': node_type['_id'],
+        'properties': {
+            'status': 'published',
+            'contentType': file_object['contentType'].split('/')[0],
+            'file': file_id
+            }
+        }
+    r = post_item('nodes', node)
+
+    return r
+
+@manager.command
+def load_migration(path):
+    import json
+    import os
+    from bson import json_util
+    if not os.path.isfile(path):
+        return "File does not exist"
+    with open(path, 'r') as infile:
+        d = json.load(infile)
+    from pymongo import MongoClient
+    client = MongoClient(MONGO_HOST, 27017)
+    db = client.eve
+
+    def commit_object(collection, f, parent=None):
+        variation_id = f.get('variation_id')
+        if variation_id:
+            del f['variation_id']
+
+        asset_id = f.get('asset_id')
+        if asset_id:
+            del f['asset_id']
+
+        node_id = f.get('node_id')
+        if node_id:
+            del f['node_id']
+
+        if parent:
+            f['parent'] = parent
+        else:
+            if f.get('parent'):
+                del f['parent']
+
+        #r = [{'_status': 'OK', '_id': 'DRY-ID'}]
+        print 'just before commit'
+        print f
+        r = post_item(collection, f)
+        if r[0]['_status'] == 'ERR':
+            print r[0]['_issues']
+
+        # Assign the Mongo ObjectID
+        f['_id'] = str(r[0]['_id'])
+        # Restore variation_id
+        if variation_id:
+            f['variation_id'] = variation_id
+        if asset_id:
+            f['asset_id'] = asset_id
+        if node_id:
+            f['node_id'] = node_id
+        print "{0} {1}".format(f['_id'], f['name'])
+        return f
+
+    # Build list of parent files
+    parent_files = [f for f in d['files'] if 'parent_asset_id' in f]
+    children_files = [f for f in d['files'] if 'parent_asset_id' not in f]
+
+    for p in parent_files:
+        # Store temp property
+        parent_asset_id = p['parent_asset_id']
+        # Remove from dict to prevent invalid submission
+        del p['parent_asset_id']
+        # Commit to database
+        p = commit_object('files', p)
+        # Restore temp property
+        p['parent_asset_id'] = parent_asset_id
+        # Find children of the current file
+        children = [c for c in children_files if c['parent'] == p['variation_id']]
+        for c in children:
+            # Commit to database with parent id
+            c = commit_object('files', c, p['_id'])
+
+
+    # Merge the dicts and replace the original one
+    d['files'] = parent_files + children_files
+
+    # Files for picture previews of folders (groups)
+    for f in d['files_group']:
+        item_id = f['item_id']
+        del f['item_id']
+        f = commit_object('files', f)
+        f['item_id'] = item_id
+
+    # Files for picture previews of assets
+    for f in d['files_asset']:
+        item_id = f['item_id']
+        del f['item_id']
+        f = commit_object('files',f)
+        f['item_id'] = item_id
+
+
+    nodes_asset = [n for n in d['nodes'] if 'asset_id' in n]
+    nodes_group = [n for n in d['nodes'] if 'node_id' in n]
+
+    def get_parent(node_id):
+        #print "Searching for {0}".format(node_id)
+        try:
+            parent = [p for p in nodes_group if p['node_id'] == node_id][0]
+        except IndexError:
+            return None
+        return parent
+
+    def traverse_nodes(parent_id):
+        parents_list = []
+        while True:
+            parent = get_parent(parent_id)
+            #print parent
+            if not parent:
+                break
+            else:
+                parents_list.append(parent['node_id'])
+                if parent.get('parent'):
+                    parent_id = parent['parent']
+                else:
+                    break
+        parents_list.reverse()
+        return parents_list
+
+    for n in nodes_asset:
+        node_type_asset = db.node_types.find_one({"name": "asset"})
+        pictures = [p for p in d['files_asset'] if p['name'] == n['picture'][:-4]]
+        if pictures:
+            n['picture'] = pictures[0]['_id']
+            print "Adding picture link {0}".format(n['picture'])
+        n['node_type'] = node_type_asset['_id']
+        # An asset node must have a parent
+        # parent = [p for p in nodes_group if p['node_id'] == n['parent']][0]
+        parents_list = traverse_nodes(n['parent'])
+
+        tree_index = 0
+        for node_id in parents_list:
+            node = [p for p in nodes_group if p['node_id'] == node_id][0]
+
+            if node.get('_id') is None:
+                node_type_group = db.node_types.find_one({"name": "group"})
+                node['node_type'] = node_type_group['_id']
+                # Assign picture to the node group
+                if node.get('picture'):
+                    picture = [p for p in d['files_group'] if p['name'] == node['picture'][:-4]][0]
+                    node['picture'] = picture['_id']
+                    print "Adding picture link to node {0}".format(node['picture'])
+                if tree_index == 0:
+                    # We are at the root of the tree (so we link to the project)
+                    parent = None
+                else:
+                    # Get the parent node id
+                    parents_list_node_id = parents_list[tree_index - 1]
+                    parent_node = [p for p in nodes_group if p['node_id'] == parents_list_node_id][0]
+                    parent = parent_node['_id']
+                print "About to commit Node"
+                commit_object('nodes', node, parent)
+            tree_index += 1
+        # Commit the asset
+        print "About to commit Asset"
+        parent_node = [p for p in nodes_group if p['node_id'] == parents_list[-1]][0]
+        asset_file = [a for a in d['files'] if a['md5'] == n['properties']['file']][0]
+        n['properties']['file'] = str(asset_file['_id'])
+        print n
+        commit_object('nodes', n, parent_node['_id'])
+
+    return
+
+
+    # New path with _
+    path = '_' + path
+    with open(path, 'w') as outfile:
+        json.dump(d, outfile, default=json_util.default)
+    return
+
 
 
 if __name__ == '__main__':

pillar/settings.py

@@ -155,7 +155,7 @@ nodes_schema = {
     'description': {
         'type': 'string',
         'minlength': 0,
-        'maxlength': 128,
+        'maxlength': 512,
     },
     'picture': {
         'type': 'objectid',
@@ -253,7 +253,6 @@ files_schema = {
     },
     'description': {
         'type': 'string',
-        'required': True,
     },
     # If the object has a parent, it is a variation of its parent. When querying
     # for a file we are going to check if the object does NOT have a parent. In
@@ -297,10 +296,6 @@ files_schema = {
         'type': 'integer',
         'required': True,
     },
-    'uploadDate': {
-        'type': 'datetime',
-        'required': True,
-    },
     'md5': {
         'type': 'string',
         'required': True,
@@ -312,7 +307,7 @@ files_schema = {
     'backend': {
         'type': 'string',
         'required': True,
-        'allowed': ["attract-web", "pillar"]
+        'allowed': ["attract-web", "pillar", "cdnsun"]
     },
     'path': {
         'type': 'string',