diff --git a/pillar/manage.py b/pillar/manage.py index 12a30837..21434aa6 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -1,9 +1,20 @@ import os from eve.methods.put import put_internal +from flask.ext.script import Manager from application import app from application import db from application import post_item -from flask.ext.script import Manager +from manage.node_types.act import node_type_act +from manage.node_types.asset import node_type_asset +from manage.node_types.blog import node_type_blog +from manage.node_types.comment import node_type_comment +from manage.node_types.group import node_type_group +from manage.node_types.post import node_type_post +from manage.node_types.project import node_type_project +from manage.node_types.scene import node_type_scene +from manage.node_types.shot import node_type_shot +from manage.node_types.storage import node_type_storage +from manage.node_types.task import node_type_task manager = Manager(app) @@ -152,679 +163,7 @@ def manage_groups(): user_email, group_name) -@manager.command -def add_groups(): - """Add permisions - """ - admin_group = { - 'name': 'admin', - 'permissions': [ - {'node_type': get_id('node_types', 'shot'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'task'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'scene'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'act'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'comment'), - 'permissions': ['GET', 'POST', 'UPDATE', 'DELETE'] - }, - ] - } - post_item('groups', admin_group) - - owner_group = { - 'name': 'owner', - 'permissions': [ - {'node_type': get_id('node_types', 'shot'), - 'permissions': ['GET', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'task'), - 'permissions': ['GET', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'scene'), - 'permissions': ['GET', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'act'), - 'permissions': ['GET', 'UPDATE', 'DELETE'] - }, - {'node_type': get_id('node_types', 'comment'), - 'permissions': ['GET', 'UPDATE', 'DELETE'] - }, - ] - } - post_item('groups', owner_group) - - world_group = { - 'name': 'world', - 'permissions': [ - {'node_type': get_id('node_types', 'shot'), - 'permissions': ['GET'] - }, - {'node_type': get_id('node_types', 'task'), - 'permissions': ['GET'] - }, - {'node_type': get_id('node_types', 'scene'), - 'permissions': ['GET'] - }, - {'node_type': get_id('node_types', 'act'), - 'permissions': ['GET'] - }, - {'node_type': get_id('node_types', 'comment'), - 'permissions': ['GET', 'POST'] - }, - ] - } - post_item('groups', world_group) - - -@manager.command -def populate_db(): - """Populate the db with sample data - """ - populate_node_types() - - def populate_node_types(old_ids={}): - shot_node_type = { - 'name': 'shot', - 'description': 'Shot Node Type, for shots', - 'dyn_schema': { - 'url': { - 'type': 'string', - }, - 'cut_in': { - 'type': 'integer' - }, - 'cut_out': { - 'type': 'integer' - }, - 'status': { - 'type': 'string', - 'allowed': [ - 'on_hold', - 'todo', - 'in_progress', - 'review', - 'final' - ], - }, - 'notes': { - 'type': 'string', - 'maxlength': 256, - }, - 'shot_group': { - 'type': 'string', - #'data_relation': { - # 'resource': 'nodes', - # 'field': '_id', - #}, - }, - }, - 'form_schema': { - 'url': {}, - 'cut_in': {}, - 'cut_out': {}, - 'status': {}, - 'notes': {}, - 'shot_group': {} - }, - 'parent': { - 'node_types': ['scene'] - } - } - - task_node_type = { - 'name': 'task', - 'description': 'Task Node Type, for tasks', - 'dyn_schema': { - 'status': { - 'type': 'string', - 'allowed': [ - 'todo', - 'in_progress', - 'on_hold', - 'approved', - 'cbb', - 'final', - 'review' - ], - 'required': True, - }, - 'filepath': { - 'type': 'string', - }, - 'revision': { - 'type': 'integer', - }, - 'owners': { - 'type': 'dict', - 'schema': { - 'users': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - } - }, - 'groups': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - } - } - } - }, - 'time': { - 'type': 'dict', - 'schema': { - 'start': { - 'type': 'datetime' - }, - 'duration': { - 'type': 'integer' - }, - 'chunks': { - 'type': 'list', - 'schema': { - 'type': 'dict', - 'schema': { - 'start': { - 'type': 'datetime', - }, - 'duration': { - 'type': 'integer', - } - } - } - }, - } - }, - 'is_conflicting' : { - 'type': 'boolean' - }, - 'is_processing' : { - 'type': 'boolean' - }, - 'is_open' : { - 'type': 'boolean' - } - - }, - 'form_schema': { - 'status': {}, - 'filepath': {}, - 'revision': {}, - 'owners': { - 'schema': { - 'users':{ - 'items': [('User', 'first_name')], - }, - 'groups': {} - } - }, - 'time': { - 'schema': { - 'start': {}, - 'duration': {}, - 'chunks': { - 'visible': False, - 'schema': { - 'start': {}, - 'duration': {} - } - } - } - }, - 'is_conflicting': {}, - 'is_open': {}, - 'is_processing': {}, - }, - 'parent': { - 'node_types': ['shot'], - } - } - - scene_node_type = { - 'name': 'scene', - 'description': 'Scene node type', - 'parent': { - "node_types": ["act"] - } - } - - act_node_type = { - 'name': 'act', - 'description': 'Act node type', - 'parent': {} - } - - node_type_project = { - 'name': 'project', - 'parent': {}, - 'description': 'The official project type', - 'dyn_schema': { - 'category': { - 'type': 'string', - 'allowed': [ - 'training', - 'film', - 'assets', - 'software', - 'game' - ], - 'required': True, - }, - 'is_private': { - 'type': 'boolean' - }, - 'url': { - 'type': 'string' - }, - 'organization': { - 'type': 'objectid', - 'nullable': True, - 'data_relation': { - 'resource': 'organizations', - 'field': '_id', - 'embeddable': True - }, - }, - 'owners': { - 'type': 'dict', - 'schema': { - 'users': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - } - }, - 'groups': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - 'data_relation': { - 'resource': 'groups', - 'field': '_id', - 'embeddable': True - } - } - } - } - }, - 'status': { - 'type': 'string', - 'allowed': [ - 'published', - 'pending', - 'deleted' - ], - }, - # Logo - 'picture_square': { - 'type': 'objectid', - 'nullable': True, - 'data_relation': { - 'resource': 'files', - 'field': '_id', - 'embeddable': True - }, - }, - # Header - 'picture_header': { - 'type': 'objectid', - 'nullable': True, - 'data_relation': { - 'resource': 'files', - 'field': '_id', - 'embeddable': True - }, - }, - # Short summary for the project - 'summary': { - 'type': 'string', - 'maxlength': 128 - }, - # Latest nodes being edited - 'nodes_latest': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - } - }, - # Featured nodes, manually added - 'nodes_featured': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - } - }, - # Latest blog posts, manually added - 'nodes_blog': { - 'type': 'list', - 'schema': { - 'type': 'objectid', - } - } - }, - 'form_schema': { - 'is_private': {}, - # TODO add group parsing - 'category': {}, - 'url': {}, - 'organization': {}, - 'picture_square': {}, - 'picture_header': {}, - 'summary': {}, - 'owners': { - 'schema': { - 'users': {}, - 'groups': { - 'items': [('Group', 'name')], - }, - } - }, - 'status': {}, - 'nodes_featured': {}, - 'nodes_latest': {}, - 'nodes_blog': {} - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - 'world': ['GET'] - } - } - - node_type_group = { - 'name': 'group', - 'description': 'Generic group node type', - 'parent': { - 'node_types': ['group', 'project'] - }, - 'dyn_schema': { - 'url': { - 'type': 'string', - }, - 'status': { - 'type': 'string', - 'allowed': [ - 'published', - 'pending', - 'deleted' - ], - }, - 'notes': { - 'type': 'string', - 'maxlength': 256, - }, - }, - 'form_schema': { - 'url': {}, - 'status': {}, - 'notes': {}, - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - 'world': ['GET'] - } - } - - node_type_asset = { - 'name': 'asset', - 'description': 'Assets for Elephants Dream', - # This data type does not have parent limitations (can be child - # of any node). An empty parent declaration is required. - 'parent': { - "node_types": ["group",] - }, - 'dyn_schema': { - 'status': { - 'type': 'string', - 'allowed': [ - 'published', - 'pending', - 'processing', - 'deleted' - ], - }, - # We expose the type of asset we point to. Usually image, video, - # zipfile, ect. - 'content_type':{ - 'type': 'string' - }, - # We point to the original file (and use it to extract any relevant - # variation useful for our scope). - 'file': { - 'type': 'objectid', - 'data_relation': { - 'resource': 'files', - 'field': '_id', - 'embeddable': True - }, - } - }, - 'form_schema': { - 'status': {}, - 'content_type': {'visible': False}, - 'file': {'visible': False}, - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - } - } - - node_type_storage = { - 'name': 'storage', - 'description': 'Entrypoint to a remote or local storage solution', - 'dyn_schema': { - # The project ID, use for lookups in the storage backend. For example - # when using Google Cloud Storage, the project id will be the name - # of the bucket. - 'project': { - 'type': 'objectid', - 'data_relation': { - 'resource': 'nodes', - 'field': '_id' - }, - }, - # The entry point in a subdirectory of the main storage for the project - 'subdir': { - 'type': 'string', - }, - # Which backend is used to store the files (gcs, pillar, bam, cdnsun) - 'backend': { - 'type': 'string', - }, - }, - 'form_schema': { - 'subdir': {}, - 'project': {}, - 'backend': {} - }, - 'parent': { - "node_types": ["group", "project"] - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - } - } - - node_type_comment = { - 'name': 'comment', - 'description': 'Comments for asset asset nodes, pages, etc.', - 'dyn_schema': { - # The actual comment content (initially Markdown format) - 'content': { - 'type': 'string', - 'minlength': 5, - }, - 'status': { - 'type': 'string', - 'allowed': [ - 'published', - 'deleted', - 'flagged', - 'edited' - ], - }, - # Total count of positive ratings (updated at every rating action) - 'rating_positive': { - 'type': 'integer', - }, - # Total count of negative ratings (updated at every rating action) - 'rating_negative': { - 'type': 'integer', - }, - # Collection of ratings, keyed by user - 'ratings': { - 'type': 'list', - 'schema': { - 'type': 'dict', - 'schema': { - 'user': { - 'type': 'objectid' - }, - 'is_positive': { - 'type': 'boolean' - }, - # Weight of the rating based on user rep and the context. - # Currently we have the following weights: - # - 1 auto null - # - 2 manual null - # - 3 auto valid - # - 4 manual valid - 'weight': { - 'type': 'integer' - } - } - } - }, - 'confidence': { - 'type': 'float' - } - - }, - 'form_schema': { - 'content': {}, - 'status': {}, - 'rating_positive': {}, - 'rating_negative': {}, - 'ratings': {}, - 'confidence': {} - }, - 'parent': { - 'node_types': ['asset',] - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - 'world': ['GET'] - } - } - - node_type_blog = { - 'name': 'blog', - 'description': 'Container for node_type post.', - 'dyn_schema': { - # Path for a custom template to be used for rendering the posts - 'template': { - 'type': 'string', - }, - 'categories' : { - 'type': 'list', - 'schema': { - 'type': 'string' - } - } - }, - 'form_schema': { - 'categories': {}, - 'template': {}, - }, - 'parent': { - 'node_types': ['project',] - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - 'world': ['GET'] - } - } - - node_type_post = { - 'name': 'post', - 'description': 'A blog post, for any project', - 'dyn_schema': { - # The blogpost content (Markdown format) - 'content': { - 'type': 'string', - 'minlength': 5, - 'maxlength': 90000, - 'required': True - }, - 'status': { - 'type': 'string', - 'allowed': [ - 'published', - 'deleted', - 'pending' - ], - 'default': 'pending' - }, - # Global categories, will be enforced to be 1 word - 'category': { - 'type': 'string', - }, - 'url': { - 'type': 'string' - } - }, - 'form_schema': { - 'content': {}, - 'status': {}, - 'category': {}, - 'url': {} - }, - 'parent': { - 'node_types': ['blog',] - }, - 'permissions': { - 'groups': [{ - 'group': app.config['ADMIN_USER_GROUP'], - 'methods': ['GET', 'PUT', 'POST'] - }], - 'users': [], - 'world': ['GET'] - } - } - - from pymongo import MongoClient @@ -852,9 +191,9 @@ def populate_node_types(old_ids={}): internal_fields = ['_id', '_etag', '_updated', '_created'] for field in internal_fields: node_type.pop(field, None) - + # Also remove permissions, since they are managed separately + node_type.pop('permissions', None) p = put_internal('node_types', node_type, **{'_id': node_id}) - print p else: print("Making the node") @@ -874,316 +213,9 @@ def populate_node_types(old_ids={}): upgrade(node_type_post, old_ids) -@manager.command -def add_file_video(): - from datetime import datetime - RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' - video = { - 'name': 'Video test', - 'description': 'Video test description', - # 'parent': 'objectid', - 'content_type': 'video/mp4', - # Duration in seconds, only if it's a video - 'duration': 50, - 'size': '720p', - 'format': 'mp4', - 'width': 1280, - 'height': 720, - 'user': '552b066b41acdf5dec4436f2', - 'length': 15000, - 'md5': 'md5', - 'filename': 'The file name', - 'backend': 'pillar', - 'path': '0000.mp4', - } - r = post_item('files', video) - return r - -@manager.command -def add_node_asset(file_id): - """Creates a node of type asset, starting from an existing file id. - :param file_id: the file id to use - :param picture_id: the picture id to use - :param group_id: the parent folder - """ - picture_id = None - parent_id = None - - from bson.objectid import ObjectId - from pymongo import MongoClient - client = MongoClient(MONGO_HOST, 27017) - db = client.eve - file_object = db.files.find_one({"_id": ObjectId(file_id)}) - node_type = db.node_types.find_one({"name": "asset"}) - - print file_object['content_type'].split('/')[0] - - node = { - 'name': file_object['name'], - 'description': file_object['description'], - #'picture': picture_id, - #'parent': parent_id, - 'user': file_object['user'], - 'node_type': node_type['_id'], - 'properties': { - 'status': 'published', - 'content_type': file_object['content_type'].split('/')[0], - 'file': file_id - } - } - r = post_item('nodes', node) - - return r - -@manager.command -def import_data(path): - import json - import pprint - from bson import json_util - if not os.path.isfile(path): - return "File does not exist" - with open(path, 'r') as infile: - d = json.load(infile) - - def commit_object(collection, f, parent=None): - variation_id = f.get('variation_id') - if variation_id: - del f['variation_id'] - - asset_id = f.get('asset_id') - if asset_id: - del f['asset_id'] - - node_id = f.get('node_id') - if node_id: - del f['node_id'] - - if parent: - f['parent'] = parent - else: - if f.get('parent'): - del f['parent'] - - #r = [{'_status': 'OK', '_id': 'DRY-ID'}] - r = post_item(collection, f) - if r[0]['_status'] == 'ERR': - print r[0]['_issues'] - print "Tried to commit the following object" - pprint.pprint(f) - - # Assign the Mongo ObjectID - f['_id'] = str(r[0]['_id']) - # Restore variation_id - if variation_id: - f['variation_id'] = variation_id - if asset_id: - f['asset_id'] = asset_id - if node_id: - f['node_id'] = node_id - try: - print "{0} {1}".format(f['_id'], f['name']) - except UnicodeEncodeError: - print "{0}".format(f['_id']) - return f - - # Build list of parent files - parent_files = [f for f in d['files'] if 'parent_asset_id' in f] - children_files = [f for f in d['files'] if 'parent_asset_id' not in f] - - for p in parent_files: - # Store temp property - parent_asset_id = p['parent_asset_id'] - # Remove from dict to prevent invalid submission - del p['parent_asset_id'] - # Commit to database - p = commit_object('files', p) - # Restore temp property - p['parent_asset_id'] = parent_asset_id - # Find children of the current file - children = [c for c in children_files if c['parent'] == p['variation_id']] - for c in children: - # Commit to database with parent id - c = commit_object('files', c, p['_id']) - - - # Merge the dicts and replace the original one - d['files'] = parent_files + children_files - - # Files for picture previews of folders (groups) - for f in d['files_group']: - item_id = f['item_id'] - del f['item_id'] - f = commit_object('files', f) - f['item_id'] = item_id - - # Files for picture previews of assets - for f in d['files_asset']: - item_id = f['item_id'] - del f['item_id'] - f = commit_object('files',f) - f['item_id'] = item_id - - - nodes_asset = [n for n in d['nodes'] if 'asset_id' in n] - nodes_group = [n for n in d['nodes'] if 'node_id' in n] - - def get_parent(node_id): - #print "Searching for {0}".format(node_id) - try: - parent = [p for p in nodes_group if p['node_id'] == node_id][0] - except IndexError: - return None - return parent - - def traverse_nodes(parent_id): - parents_list = [] - while True: - parent = get_parent(parent_id) - #print parent - if not parent: - break - else: - parents_list.append(parent['node_id']) - if parent.get('parent'): - parent_id = parent['parent'] - else: - break - parents_list.reverse() - return parents_list - - for n in nodes_asset: - node_type_asset = db.node_types.find_one({"name": "asset"}) - if n.get('picture'): - filename = os.path.splitext(n['picture'])[0] - pictures = [p for p in d['files_asset'] if p['name'] == filename] - if pictures: - n['picture'] = pictures[0]['_id'] - print "Adding picture link {0}".format(n['picture']) - n['node_type'] = node_type_asset['_id'] - # An asset node must have a parent - # parent = [p for p in nodes_group if p['node_id'] == n['parent']][0] - parents_list = traverse_nodes(n['parent']) - - tree_index = 0 - for node_id in parents_list: - node = [p for p in nodes_group if p['node_id'] == node_id][0] - - if node.get('_id') is None: - node_type_group = db.node_types.find_one({"name": "group"}) - node['node_type'] = node_type_group['_id'] - # Assign picture to the node group - if node.get('picture'): - filename = os.path.splitext(node['picture'])[0] - picture = [p for p in d['files_group'] if p['name'] == filename][0] - node['picture'] = picture['_id'] - print "Adding picture link to node {0}".format(node['picture']) - if tree_index == 0: - # We are at the root of the tree (so we link to the project) - node_type_project = db.node_types.find_one({"name": "project"}) - node['node_type'] = node_type_project['_id'] - parent = None - if node['properties'].get('picture_square'): - filename = os.path.splitext(node['properties']['picture_square'])[0] - picture = [p for p in d['files_group'] if p['name'] == filename][0] - node['properties']['picture_square'] = picture['_id'] - print "Adding picture_square link to node" - if node['properties'].get('picture_header'): - filename = os.path.splitext(node['properties']['picture_header'])[0] - picture = [p for p in d['files_group'] if p['name'] == filename][0] - node['properties']['picture_header'] = picture['_id'] - print "Adding picture_header link to node" - else: - # Get the parent node id - parents_list_node_id = parents_list[tree_index - 1] - parent_node = [p for p in nodes_group if p['node_id'] == parents_list_node_id][0] - parent = parent_node['_id'] - print "About to commit Node" - commit_object('nodes', node, parent) - tree_index += 1 - # Commit the asset - print "About to commit Asset {0}".format(n['asset_id']) - parent_node = [p for p in nodes_group if p['node_id'] == parents_list[-1]][0] - try: - asset_file = [a for a in d['files'] if a['md5'] == n['properties']['file']][0] - n['properties']['file'] = str(asset_file['_id']) - commit_object('nodes', n, parent_node['_id']) - except IndexError: - pass - - return - - - # New path with _ - path = '_' + path - with open(path, 'w') as outfile: - json.dump(d, outfile, default=json_util.default) - return - -@manager.command -def make_thumbnails(): - from bson.objectid import ObjectId - from application.modules.file_storage import build_thumbnails - import shutil - files = db.files.find() - for f in files: - if f['content_type'].split('/')[0] == 'image': - - if '-' in f['path']: - pass - #print "Skipping {0}".format(f['path']) - elif f['backend'] == 'pillar': - a = db.files.find({'parent': ObjectId(f['_id'])}) - a_count = 0 - for b in a: - a_count += 1 - if a_count == 0: - file_path = f['path'] - print "Building {0}".format(file_path) - file_full_path = os.path.join( - app.config['STORAGE_DIR'], file_path) - if not os.path.exists(file_full_path): - src_shared_full_path = os.path.join( - app.config['SHARED_DIR'], 'files', file_path[3:]) - print src_shared_full_path - - dst_root_dir = os.path.join( - app.config['STORAGE_DIR'], file_path[:3]) - dst_file_full_path_ = os.path.join( - app.config['STORAGE_DIR'], file_path) - if not os.path.exists(dst_root_dir): - os.makedirs(dst_root_dir) - shutil.copy(src_shared_full_path, - dst_file_full_path_) - print dst_file_full_path_ - else: - t = build_thumbnails(file_path=file_path) - print t - - #t = build_thumbnails(file_path=f['path']) - #print t - else: - pass - -@manager.command -def add_node_permissions(): - import codecs - import sys - UTF8Writer = codecs.getwriter('utf8') - sys.stdout = UTF8Writer(sys.stdout) - nodes_collection = app.data.driver.db['nodes'] - node_types_collection = app.data.driver.db['node_types'] - nodes = nodes_collection.find() - for node in nodes: - print u"{0}".format(node['name']) - if 'permissions' not in node: - node_type = node_types_collection.find_one(node['node_type']) - # nodes_collection.update({'_id': node['_id']}, - # {"$set": {'permissions': node_type['permissions']}}) - print node['_id'] - break - @manager.command def add_parent_to_nodes(): + """Find the parent of any node in the nodes collection""" import codecs import sys from bson.objectid import ObjectId diff --git a/pillar/manage/__init__.py b/pillar/manage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pillar/manage/import_data.py b/pillar/manage/import_data.py new file mode 100644 index 00000000..21b42f93 --- /dev/null +++ b/pillar/manage/import_data.py @@ -0,0 +1,182 @@ +def import_data(path): + import json + import pprint + from bson import json_util + if not os.path.isfile(path): + return "File does not exist" + with open(path, 'r') as infile: + d = json.load(infile) + + def commit_object(collection, f, parent=None): + variation_id = f.get('variation_id') + if variation_id: + del f['variation_id'] + + asset_id = f.get('asset_id') + if asset_id: + del f['asset_id'] + + node_id = f.get('node_id') + if node_id: + del f['node_id'] + + if parent: + f['parent'] = parent + else: + if f.get('parent'): + del f['parent'] + + #r = [{'_status': 'OK', '_id': 'DRY-ID'}] + r = post_item(collection, f) + if r[0]['_status'] == 'ERR': + print r[0]['_issues'] + print "Tried to commit the following object" + pprint.pprint(f) + + # Assign the Mongo ObjectID + f['_id'] = str(r[0]['_id']) + # Restore variation_id + if variation_id: + f['variation_id'] = variation_id + if asset_id: + f['asset_id'] = asset_id + if node_id: + f['node_id'] = node_id + try: + print "{0} {1}".format(f['_id'], f['name']) + except UnicodeEncodeError: + print "{0}".format(f['_id']) + return f + + # Build list of parent files + parent_files = [f for f in d['files'] if 'parent_asset_id' in f] + children_files = [f for f in d['files'] if 'parent_asset_id' not in f] + + for p in parent_files: + # Store temp property + parent_asset_id = p['parent_asset_id'] + # Remove from dict to prevent invalid submission + del p['parent_asset_id'] + # Commit to database + p = commit_object('files', p) + # Restore temp property + p['parent_asset_id'] = parent_asset_id + # Find children of the current file + children = [c for c in children_files if c['parent'] == p['variation_id']] + for c in children: + # Commit to database with parent id + c = commit_object('files', c, p['_id']) + + + # Merge the dicts and replace the original one + d['files'] = parent_files + children_files + + # Files for picture previews of folders (groups) + for f in d['files_group']: + item_id = f['item_id'] + del f['item_id'] + f = commit_object('files', f) + f['item_id'] = item_id + + # Files for picture previews of assets + for f in d['files_asset']: + item_id = f['item_id'] + del f['item_id'] + f = commit_object('files',f) + f['item_id'] = item_id + + + nodes_asset = [n for n in d['nodes'] if 'asset_id' in n] + nodes_group = [n for n in d['nodes'] if 'node_id' in n] + + def get_parent(node_id): + #print "Searching for {0}".format(node_id) + try: + parent = [p for p in nodes_group if p['node_id'] == node_id][0] + except IndexError: + return None + return parent + + def traverse_nodes(parent_id): + parents_list = [] + while True: + parent = get_parent(parent_id) + #print parent + if not parent: + break + else: + parents_list.append(parent['node_id']) + if parent.get('parent'): + parent_id = parent['parent'] + else: + break + parents_list.reverse() + return parents_list + + for n in nodes_asset: + node_type_asset = db.node_types.find_one({"name": "asset"}) + if n.get('picture'): + filename = os.path.splitext(n['picture'])[0] + pictures = [p for p in d['files_asset'] if p['name'] == filename] + if pictures: + n['picture'] = pictures[0]['_id'] + print "Adding picture link {0}".format(n['picture']) + n['node_type'] = node_type_asset['_id'] + # An asset node must have a parent + # parent = [p for p in nodes_group if p['node_id'] == n['parent']][0] + parents_list = traverse_nodes(n['parent']) + + tree_index = 0 + for node_id in parents_list: + node = [p for p in nodes_group if p['node_id'] == node_id][0] + + if node.get('_id') is None: + node_type_group = db.node_types.find_one({"name": "group"}) + node['node_type'] = node_type_group['_id'] + # Assign picture to the node group + if node.get('picture'): + filename = os.path.splitext(node['picture'])[0] + picture = [p for p in d['files_group'] if p['name'] == filename][0] + node['picture'] = picture['_id'] + print "Adding picture link to node {0}".format(node['picture']) + if tree_index == 0: + # We are at the root of the tree (so we link to the project) + node_type_project = db.node_types.find_one({"name": "project"}) + node['node_type'] = node_type_project['_id'] + parent = None + if node['properties'].get('picture_square'): + filename = os.path.splitext(node['properties']['picture_square'])[0] + picture = [p for p in d['files_group'] if p['name'] == filename][0] + node['properties']['picture_square'] = picture['_id'] + print "Adding picture_square link to node" + if node['properties'].get('picture_header'): + filename = os.path.splitext(node['properties']['picture_header'])[0] + picture = [p for p in d['files_group'] if p['name'] == filename][0] + node['properties']['picture_header'] = picture['_id'] + print "Adding picture_header link to node" + else: + # Get the parent node id + parents_list_node_id = parents_list[tree_index - 1] + parent_node = [p for p in nodes_group if p['node_id'] == parents_list_node_id][0] + parent = parent_node['_id'] + print "About to commit Node" + commit_object('nodes', node, parent) + tree_index += 1 + # Commit the asset + print "About to commit Asset {0}".format(n['asset_id']) + parent_node = [p for p in nodes_group if p['node_id'] == parents_list[-1]][0] + try: + asset_file = [a for a in d['files'] if a['md5'] == n['properties']['file']][0] + n['properties']['file'] = str(asset_file['_id']) + commit_object('nodes', n, parent_node['_id']) + except IndexError: + pass + + return + + + # New path with _ + path = '_' + path + with open(path, 'w') as outfile: + json.dump(d, outfile, default=json_util.default) + return diff --git a/pillar/manage/node_types/__init__.py b/pillar/manage/node_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pillar/manage/node_types/act.py b/pillar/manage/node_types/act.py new file mode 100644 index 00000000..a50ac19a --- /dev/null +++ b/pillar/manage/node_types/act.py @@ -0,0 +1,5 @@ +node_type_act = { + 'name': 'act', + 'description': 'Act node type', + 'parent': {} +} diff --git a/pillar/manage/node_types/asset.py b/pillar/manage/node_types/asset.py new file mode 100644 index 00000000..5df5a1ef --- /dev/null +++ b/pillar/manage/node_types/asset.py @@ -0,0 +1,47 @@ +node_type_asset = { + 'name': 'asset', + 'description': 'Basic Asset Type', + # This data type does not have parent limitations (can be child + # of any node). An empty parent declaration is required. + 'parent': { + "node_types": ["group",] + }, + 'dyn_schema': { + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'pending', + 'processing', + 'deleted' + ], + }, + # We expose the type of asset we point to. Usually image, video, + # zipfile, ect. + 'content_type':{ + 'type': 'string' + }, + # We point to the original file (and use it to extract any relevant + # variation useful for our scope). + 'file': { + 'type': 'objectid', + 'data_relation': { + 'resource': 'files', + 'field': '_id', + 'embeddable': True + }, + } + }, + 'form_schema': { + 'status': {}, + 'content_type': {'visible': False}, + 'file': {'visible': False}, + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + } +} diff --git a/pillar/manage/node_types/blog.py b/pillar/manage/node_types/blog.py new file mode 100644 index 00000000..8718ed9c --- /dev/null +++ b/pillar/manage/node_types/blog.py @@ -0,0 +1,31 @@ +node_type_blog = { + 'name': 'blog', + 'description': 'Container for node_type post.', + 'dyn_schema': { + # Path for a custom template to be used for rendering the posts + 'template': { + 'type': 'string', + }, + 'categories' : { + 'type': 'list', + 'schema': { + 'type': 'string' + } + } + }, + 'form_schema': { + 'categories': {}, + 'template': {}, + }, + 'parent': { + 'node_types': ['project',] + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + # 'world': ['GET'] + } +} diff --git a/pillar/manage/node_types/comment.py b/pillar/manage/node_types/comment.py new file mode 100644 index 00000000..7116a8af --- /dev/null +++ b/pillar/manage/node_types/comment.py @@ -0,0 +1,75 @@ +node_type_comment = { + 'name': 'comment', + 'description': 'Comments for asset asset nodes, pages, etc.', + 'dyn_schema': { + # The actual comment content (initially Markdown format) + 'content': { + 'type': 'string', + 'minlength': 5, + }, + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'deleted', + 'flagged', + 'edited' + ], + }, + # Total count of positive ratings (updated at every rating action) + 'rating_positive': { + 'type': 'integer', + }, + # Total count of negative ratings (updated at every rating action) + 'rating_negative': { + 'type': 'integer', + }, + # Collection of ratings, keyed by user + 'ratings': { + 'type': 'list', + 'schema': { + 'type': 'dict', + 'schema': { + 'user': { + 'type': 'objectid' + }, + 'is_positive': { + 'type': 'boolean' + }, + # Weight of the rating based on user rep and the context. + # Currently we have the following weights: + # - 1 auto null + # - 2 manual null + # - 3 auto valid + # - 4 manual valid + 'weight': { + 'type': 'integer' + } + } + } + }, + 'confidence': { + 'type': 'float' + } + + }, + 'form_schema': { + 'content': {}, + 'status': {}, + 'rating_positive': {}, + 'rating_negative': {}, + 'ratings': {}, + 'confidence': {} + }, + 'parent': { + 'node_types': ['asset',] + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + # 'world': ['GET'] + } +} diff --git a/pillar/manage/node_types/group.py b/pillar/manage/node_types/group.py new file mode 100644 index 00000000..5810bd20 --- /dev/null +++ b/pillar/manage/node_types/group.py @@ -0,0 +1,37 @@ +node_type_group = { + 'name': 'group', + 'description': 'Generic group node type', + 'parent': { + 'node_types': ['group', 'project'] + }, + 'dyn_schema': { + 'url': { + 'type': 'string', + }, + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'pending', + 'deleted' + ], + }, + 'notes': { + 'type': 'string', + 'maxlength': 256, + }, + }, + 'form_schema': { + 'url': {}, + 'status': {}, + 'notes': {}, + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + # 'world': ['GET'] + } +} diff --git a/pillar/manage/node_types/post.py b/pillar/manage/node_types/post.py new file mode 100644 index 00000000..1419e1db --- /dev/null +++ b/pillar/manage/node_types/post.py @@ -0,0 +1,47 @@ +node_type_post = { + 'name': 'post', + 'description': 'A blog post, for any project', + 'dyn_schema': { + # The blogpost content (Markdown format) + 'content': { + 'type': 'string', + 'minlength': 5, + 'maxlength': 90000, + 'required': True + }, + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'deleted', + 'pending' + ], + 'default': 'pending' + }, + # Global categories, will be enforced to be 1 word + 'category': { + 'type': 'string', + }, + 'url': { + 'type': 'string' + } + }, + 'form_schema': { + 'content': {}, + 'status': {}, + 'category': {}, + 'url': {} + }, + 'parent': { + 'node_types': ['blog',] + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + # 'world': ['GET'] + } +} + diff --git a/pillar/manage/node_types/project.py b/pillar/manage/node_types/project.py new file mode 100644 index 00000000..4759619c --- /dev/null +++ b/pillar/manage/node_types/project.py @@ -0,0 +1,139 @@ +node_type_project = { + 'name': 'project', + 'parent': {}, + 'description': 'The official project type', + 'dyn_schema': { + 'category': { + 'type': 'string', + 'allowed': [ + 'training', + 'film', + 'assets', + 'software', + 'game' + ], + 'required': True, + }, + 'is_private': { + 'type': 'boolean' + }, + 'url': { + 'type': 'string' + }, + 'organization': { + 'type': 'objectid', + 'nullable': True, + 'data_relation': { + 'resource': 'organizations', + 'field': '_id', + 'embeddable': True + }, + }, + 'owners': { + 'type': 'dict', + 'schema': { + 'users': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + 'groups': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + 'data_relation': { + 'resource': 'groups', + 'field': '_id', + 'embeddable': True + } + } + } + } + }, + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'pending', + 'deleted' + ], + }, + # Logo + 'picture_square': { + 'type': 'objectid', + 'nullable': True, + 'data_relation': { + 'resource': 'files', + 'field': '_id', + 'embeddable': True + }, + }, + # Header + 'picture_header': { + 'type': 'objectid', + 'nullable': True, + 'data_relation': { + 'resource': 'files', + 'field': '_id', + 'embeddable': True + }, + }, + # Short summary for the project + 'summary': { + 'type': 'string', + 'maxlength': 128 + }, + # Latest nodes being edited + 'nodes_latest': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + # Featured nodes, manually added + 'nodes_featured': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + # Latest blog posts, manually added + 'nodes_blog': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + } + }, + 'form_schema': { + 'is_private': {}, + # TODO add group parsing + 'category': {}, + 'url': {}, + 'organization': {}, + 'picture_square': {}, + 'picture_header': {}, + 'summary': {}, + 'owners': { + 'schema': { + 'users': {}, + 'groups': { + 'items': [('Group', 'name')], + }, + } + }, + 'status': {}, + 'nodes_featured': {}, + 'nodes_latest': {}, + 'nodes_blog': {} + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + # 'world': ['GET'] + } +} diff --git a/pillar/manage/node_types/scene.py b/pillar/manage/node_types/scene.py new file mode 100644 index 00000000..ca710ed3 --- /dev/null +++ b/pillar/manage/node_types/scene.py @@ -0,0 +1,7 @@ +node_type_scene = { + 'name': 'scene', + 'description': 'Scene node type', + 'parent': { + "node_types": ["act"] + } +} diff --git a/pillar/manage/node_types/shot.py b/pillar/manage/node_types/shot.py new file mode 100644 index 00000000..b168a8c3 --- /dev/null +++ b/pillar/manage/node_types/shot.py @@ -0,0 +1,47 @@ +node_type_shot = { + 'name': 'shot', + 'description': 'Shot Node Type, for shots', + 'dyn_schema': { + 'url': { + 'type': 'string', + }, + 'cut_in': { + 'type': 'integer' + }, + 'cut_out': { + 'type': 'integer' + }, + 'status': { + 'type': 'string', + 'allowed': [ + 'on_hold', + 'todo', + 'in_progress', + 'review', + 'final' + ], + }, + 'notes': { + 'type': 'string', + 'maxlength': 256, + }, + 'shot_group': { + 'type': 'string', + #'data_relation': { + # 'resource': 'nodes', + # 'field': '_id', + #}, + }, + }, + 'form_schema': { + 'url': {}, + 'cut_in': {}, + 'cut_out': {}, + 'status': {}, + 'notes': {}, + 'shot_group': {} + }, + 'parent': { + 'node_types': ['scene'] + } +} diff --git a/pillar/manage/node_types/storage.py b/pillar/manage/node_types/storage.py new file mode 100644 index 00000000..8e273a2a --- /dev/null +++ b/pillar/manage/node_types/storage.py @@ -0,0 +1,39 @@ +node_type_storage = { + 'name': 'storage', + 'description': 'Entrypoint to a remote or local storage solution', + 'dyn_schema': { + # The project ID, use for lookups in the storage backend. For example + # when using Google Cloud Storage, the project id will be the name + # of the bucket. + 'project': { + 'type': 'objectid', + 'data_relation': { + 'resource': 'nodes', + 'field': '_id' + }, + }, + # The entry point in a subdirectory of the main storage for the project + 'subdir': { + 'type': 'string', + }, + # Which backend is used to store the files (gcs, pillar, bam, cdnsun) + 'backend': { + 'type': 'string', + }, + }, + 'form_schema': { + 'subdir': {}, + 'project': {}, + 'backend': {} + }, + 'parent': { + "node_types": ["group", "project"] + }, + 'permissions': { + # 'groups': [{ + # 'group': app.config['ADMIN_USER_GROUP'], + # 'methods': ['GET', 'PUT', 'POST'] + # }], + # 'users': [], + } +} diff --git a/pillar/manage/node_types/task.py b/pillar/manage/node_types/task.py new file mode 100644 index 00000000..2b83fdd4 --- /dev/null +++ b/pillar/manage/node_types/task.py @@ -0,0 +1,109 @@ +node_type_task = { + 'name': 'task', + 'description': 'Task Node Type, for tasks', + 'dyn_schema': { + 'status': { + 'type': 'string', + 'allowed': [ + 'todo', + 'in_progress', + 'on_hold', + 'approved', + 'cbb', + 'final', + 'review' + ], + 'required': True, + }, + 'filepath': { + 'type': 'string', + }, + 'revision': { + 'type': 'integer', + }, + 'owners': { + 'type': 'dict', + 'schema': { + 'users': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + 'groups': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + } + } + }, + 'time': { + 'type': 'dict', + 'schema': { + 'start': { + 'type': 'datetime' + }, + 'duration': { + 'type': 'integer' + }, + 'chunks': { + 'type': 'list', + 'schema': { + 'type': 'dict', + 'schema': { + 'start': { + 'type': 'datetime', + }, + 'duration': { + 'type': 'integer', + } + } + } + }, + } + }, + 'is_conflicting' : { + 'type': 'boolean' + }, + 'is_processing' : { + 'type': 'boolean' + }, + 'is_open' : { + 'type': 'boolean' + } + + }, + 'form_schema': { + 'status': {}, + 'filepath': {}, + 'revision': {}, + 'owners': { + 'schema': { + 'users':{ + 'items': [('User', 'first_name')], + }, + 'groups': {} + } + }, + 'time': { + 'schema': { + 'start': {}, + 'duration': {}, + 'chunks': { + 'visible': False, + 'schema': { + 'start': {}, + 'duration': {} + } + } + } + }, + 'is_conflicting': {}, + 'is_open': {}, + 'is_processing': {}, + }, + 'parent': { + 'node_types': ['shot'], + } +}