diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index cd2b569f..a193f3ff 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -224,6 +224,8 @@ def post_item(entry, data): app = Eve(validator=ValidateCustomFields, auth=CustomTokenAuth) +import config +app.config.from_object(config.Deployment) def global_validation(): setattr(g, 'token_data', validate_token()) @@ -290,13 +292,14 @@ def post_GET_user(request, payload): app.on_post_GET_users += post_GET_user +from utils import hash_file_path # Hook to check the backend of a file resource, to build an appropriate link # that can be used by the client to retrieve the actual file. def generate_link(backend, path): if backend == 'pillar': link = url_for('file_server.index', file_name=path, _external=True) elif backend == 'cdnsun': - pass + link = hash_file_path(path, None) else: link = None return link diff --git a/pillar/application/utils.py b/pillar/application/utils.py new file mode 100644 index 00000000..783d5a7d --- /dev/null +++ b/pillar/application/utils.py @@ -0,0 +1,39 @@ +import datetime +from hashlib import md5 +from application import app + +def hash_file_path(file_path, expiry_timestamp=None): + service_domain = app.config['CDN_SERVICE_DOMAIN'] + domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER'] + asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \ + '://' + \ + service_domain + \ + domain_subfolder + \ + file_path + + if app.config['CDN_USE_URL_SIGNING']: + + url_signing_key = app.config['CDN_URL_SIGNING_KEY'] + if not file_path.startswith('/'): + file_path = '/' + file_path; + + hash_string = domain_subfolder + file_path + url_signing_key; + + if not expiry_timestamp: + expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=12) + expiry_timestamp = expiry_timestamp.strftime('%s') + + hash_string = expiry_timestamp + hash_string; + + expiry_timestamp = "," + str(expiry_timestamp); + + hashed_file_path = md5(hash_string).digest().encode('base64')[:-1] + hashed_file_path = hashed_file_path.replace('+', '-') + hashed_file_path = hashed_file_path.replace('/', '_') + + asset_url = asset_url + \ + '?secure=' + \ + hashed_file_path + \ + expiry_timestamp + + return asset_url diff --git a/pillar/config.py.example b/pillar/config.py.example index fc3d258d..c5a43a39 100644 --- a/pillar/config.py.example +++ b/pillar/config.py.example @@ -1,5 +1,4 @@ import os - class Config(object): # Configured for GMAIL MAIL_SERVER = '' @@ -9,20 +8,13 @@ class Config(object): MAIL_PASSWORD = '' DEFAULT_MAIL_SENDER = '' - # Flask-Security setup - SECURITY_LOGIN_WITHOUT_CONFIRMATION = True - SECURITY_REGISTERABLE = True - SECURITY_RECOVERABLE = True - SECURITY_CHANGEABLE = True - SECUIRTY_POST_LOGIN = '/' - SECURITY_PASSWORD_HASH = '' - SECURITY_PASSWORD_SALT = '' - SECURITY_EMAIL_SENDER = '' - class Development(Config): - SECRET_KEY = '' - HOST = '0.0.0.0' - PORT = 5000 - DEBUG = True FILE_STORAGE = '{0}/application/static/storage'.format( os.path.join(os.path.dirname(__file__))) + CDN_USE_URL_SIGNING = False + CDN_SERVICE_DOMAIN_PROTOCOL = 'https' + CDN_SERVICE_DOMAIN = '' + CDN_CONTENT_SUBFOLDER = '' + CDN_URL_SIGNING_KEY = '' + CDN_STORAGE_USER = '' + CDN_STORAGE_ADDRESS = '' diff --git a/pillar/manage.py b/pillar/manage.py index 11f69cf6..4806d594 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -241,7 +241,7 @@ def add_groups(): @manager.command -def populate_db_test(): +def populate_db(): """Populate the db with sample data """ populate_node_types() @@ -593,7 +593,9 @@ def populate_node_types(old_ids={}): 'description': 'Assets for Elephants Dream', # This data type does not have parent limitations (can be child # of any node). An empty parent declaration is required. - 'parent': {}, + 'parent': { + "node_types": ["group",] + }, 'dyn_schema': { 'status': { 'type': 'string', @@ -605,7 +607,7 @@ def populate_node_types(old_ids={}): }, # We expose the type of asset we point to. Usually image, video, # zipfile, ect. - 'contentType':{ + 'content_type':{ 'type': 'string' }, # We point to the original file (and use it to extract any relevant @@ -621,7 +623,7 @@ def populate_node_types(old_ids={}): }, 'form_schema': { 'status': {}, - 'contentType': {}, + 'content_type': {}, 'file': {}, } } @@ -662,33 +664,9 @@ def populate_node_types(old_ids={}): # upgrade(project_node_type, old_ids) upgrade(asset_node_type, old_ids) -@manager.command -def add_group(): - owners_group = { - 'name': 'owners', - 'permissions': [ - {'node_type': get_id('node_types', 'shot'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'task'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'scene'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'act'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'comment'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - ] - } - post_item('groups', world_group) - @manager.command -def add_file(): +def add_file_video(): from datetime import datetime RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' video = { @@ -704,15 +682,217 @@ def add_file(): 'height': 720, 'user': '552b066b41acdf5dec4436f2', 'length': 15000, - 'uploadDate': datetime.strftime( - datetime.now(), RFC1123_DATE_FORMAT), 'md5': 'md5', 'filename': 'The file name', 'backend': 'pillar', 'path': '0000.mp4', } r = post_item('files', video) - print r + return r + +@manager.command +def add_node_asset(file_id): + """Creates a node of type asset, starting from an existing file id. + :param file_id: the file id to use + :param picture_id: the picture id to use + :param group_id: the parent folder + """ + picture_id = None + parent_id = None + + from bson.objectid import ObjectId + from pymongo import MongoClient + client = MongoClient(MONGO_HOST, 27017) + db = client.eve + file_object = db.files.find_one({"_id": ObjectId(file_id)}) + node_type = db.node_types.find_one({"name": "asset"}) + + print file_object['contentType'].split('/')[0] + + node = { + 'name': file_object['name'], + 'description': file_object['description'], + #'picture': picture_id, + #'parent': parent_id, + 'user': file_object['user'], + 'node_type': node_type['_id'], + 'properties': { + 'status': 'published', + 'contentType': file_object['contentType'].split('/')[0], + 'file': file_id + } + } + r = post_item('nodes', node) + + return r + +@manager.command +def load_migration(path): + import json + import os + from bson import json_util + if not os.path.isfile(path): + return "File does not exist" + with open(path, 'r') as infile: + d = json.load(infile) + from pymongo import MongoClient + client = MongoClient(MONGO_HOST, 27017) + db = client.eve + + def commit_object(collection, f, parent=None): + variation_id = f.get('variation_id') + if variation_id: + del f['variation_id'] + + asset_id = f.get('asset_id') + if asset_id: + del f['asset_id'] + + node_id = f.get('node_id') + if node_id: + del f['node_id'] + + if parent: + f['parent'] = parent + else: + if f.get('parent'): + del f['parent'] + + #r = [{'_status': 'OK', '_id': 'DRY-ID'}] + print 'just before commit' + print f + r = post_item(collection, f) + if r[0]['_status'] == 'ERR': + print r[0]['_issues'] + + # Assign the Mongo ObjectID + f['_id'] = str(r[0]['_id']) + # Restore variation_id + if variation_id: + f['variation_id'] = variation_id + if asset_id: + f['asset_id'] = asset_id + if node_id: + f['node_id'] = node_id + print "{0} {1}".format(f['_id'], f['name']) + return f + + # Build list of parent files + parent_files = [f for f in d['files'] if 'parent_asset_id' in f] + children_files = [f for f in d['files'] if 'parent_asset_id' not in f] + + for p in parent_files: + # Store temp property + parent_asset_id = p['parent_asset_id'] + # Remove from dict to prevent invalid submission + del p['parent_asset_id'] + # Commit to database + p = commit_object('files', p) + # Restore temp property + p['parent_asset_id'] = parent_asset_id + # Find children of the current file + children = [c for c in children_files if c['parent'] == p['variation_id']] + for c in children: + # Commit to database with parent id + c = commit_object('files', c, p['_id']) + + + # Merge the dicts and replace the original one + d['files'] = parent_files + children_files + + # Files for picture previews of folders (groups) + for f in d['files_group']: + item_id = f['item_id'] + del f['item_id'] + f = commit_object('files', f) + f['item_id'] = item_id + + # Files for picture previews of assets + for f in d['files_asset']: + item_id = f['item_id'] + del f['item_id'] + f = commit_object('files',f) + f['item_id'] = item_id + + + nodes_asset = [n for n in d['nodes'] if 'asset_id' in n] + nodes_group = [n for n in d['nodes'] if 'node_id' in n] + + def get_parent(node_id): + #print "Searching for {0}".format(node_id) + try: + parent = [p for p in nodes_group if p['node_id'] == node_id][0] + except IndexError: + return None + return parent + + def traverse_nodes(parent_id): + parents_list = [] + while True: + parent = get_parent(parent_id) + #print parent + if not parent: + break + else: + parents_list.append(parent['node_id']) + if parent.get('parent'): + parent_id = parent['parent'] + else: + break + parents_list.reverse() + return parents_list + + for n in nodes_asset: + node_type_asset = db.node_types.find_one({"name": "asset"}) + pictures = [p for p in d['files_asset'] if p['name'] == n['picture'][:-4]] + if pictures: + n['picture'] = pictures[0]['_id'] + print "Adding picture link {0}".format(n['picture']) + n['node_type'] = node_type_asset['_id'] + # An asset node must have a parent + # parent = [p for p in nodes_group if p['node_id'] == n['parent']][0] + parents_list = traverse_nodes(n['parent']) + + tree_index = 0 + for node_id in parents_list: + node = [p for p in nodes_group if p['node_id'] == node_id][0] + + if node.get('_id') is None: + node_type_group = db.node_types.find_one({"name": "group"}) + node['node_type'] = node_type_group['_id'] + # Assign picture to the node group + if node.get('picture'): + picture = [p for p in d['files_group'] if p['name'] == node['picture'][:-4]][0] + node['picture'] = picture['_id'] + print "Adding picture link to node {0}".format(node['picture']) + if tree_index == 0: + # We are at the root of the tree (so we link to the project) + parent = None + else: + # Get the parent node id + parents_list_node_id = parents_list[tree_index - 1] + parent_node = [p for p in nodes_group if p['node_id'] == parents_list_node_id][0] + parent = parent_node['_id'] + print "About to commit Node" + commit_object('nodes', node, parent) + tree_index += 1 + # Commit the asset + print "About to commit Asset" + parent_node = [p for p in nodes_group if p['node_id'] == parents_list[-1]][0] + asset_file = [a for a in d['files'] if a['md5'] == n['properties']['file']][0] + n['properties']['file'] = str(asset_file['_id']) + print n + commit_object('nodes', n, parent_node['_id']) + + return + + + # New path with _ + path = '_' + path + with open(path, 'w') as outfile: + json.dump(d, outfile, default=json_util.default) + return + if __name__ == '__main__': diff --git a/pillar/settings.py b/pillar/settings.py index 3800bfbf..a0c6b7e2 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -155,7 +155,7 @@ nodes_schema = { 'description': { 'type': 'string', 'minlength': 0, - 'maxlength': 128, + 'maxlength': 512, }, 'picture': { 'type': 'objectid', @@ -253,7 +253,6 @@ files_schema = { }, 'description': { 'type': 'string', - 'required': True, }, # If the object has a parent, it is a variation of its parent. When querying # for a file we are going to check if the object does NOT have a parent. In @@ -297,10 +296,6 @@ files_schema = { 'type': 'integer', 'required': True, }, - 'uploadDate': { - 'type': 'datetime', - 'required': True, - }, 'md5': { 'type': 'string', 'required': True, @@ -312,7 +307,7 @@ files_schema = { 'backend': { 'type': 'string', 'required': True, - 'allowed': ["attract-web", "pillar"] + 'allowed': ["attract-web", "pillar", "cdnsun"] }, 'path': { 'type': 'string',