From e295165864102325f250a58f21051e42eadafed7 Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Mon, 25 Jan 2016 16:32:50 +0100 Subject: [PATCH] Introducing Projects We are now using a more document-based approach to define projects. In the new projects collection we store the definition of a project and embed the node_types. This allows for custom node_types for every single project. This change has a certain impact on the custom validators, as well as the permission computation. Further, Cerberus 0.9.1 is required in order to properly support the allow_unknown statements in the projects_schema definition. --- pillar/application/__init__.py | 29 ++-- pillar/manage.py | 62 +++++++- pillar/manage/node_types/act.py | 2 +- pillar/manage/node_types/asset.py | 4 +- pillar/manage/node_types/blog.py | 4 +- pillar/manage/node_types/comment.py | 4 +- pillar/manage/node_types/group.py | 6 +- pillar/manage/node_types/group_texture.py | 4 +- pillar/manage/node_types/post.py | 4 +- pillar/manage/node_types/scene.py | 4 +- pillar/manage/node_types/shot.py | 4 +- pillar/manage/node_types/storage.py | 4 +- pillar/manage/node_types/task.py | 4 +- pillar/manage/node_types/texture.py | 4 +- pillar/settings.py | 174 ++++++++++++++++++++-- 15 files changed, 256 insertions(+), 57 deletions(-) diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index f52e0ef9..14312e5c 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -195,16 +195,16 @@ class ValidateCustomFields(Validator): return properties def _validate_valid_properties(self, valid_properties, field, value): - node_types = app.data.driver.db['node_types'] - lookup = {} - lookup['_id'] = ObjectId(self.document['node_type']) - node_type = node_types.find_one(lookup) - + projects_collection = app.data.driver.db['projects'] + lookup = {'_id': ObjectId(self.document['project'])} + project = projects_collection.find_one(lookup) + node_type = next( + (item for item in project['node_types'] if item.get('name') \ + and item['name'] == self.document['node_type']), None) try: value = self.convert_properties(value, node_type['dyn_schema']) except Exception, e: print ("Error converting: {0}".format(e)) - #print (value) v = Validator(node_type['dyn_schema']) val = v.validate(value) @@ -309,7 +309,6 @@ def check_permissions(resource, method, append_allowed_methods=False): resource_permissions = resource['permissions'] else: resource_permissions = None - if 'node_type' in resource: if type(resource['node_type']) is dict: # If the node_type is embedded in the document, extract permissions @@ -318,8 +317,18 @@ def check_permissions(resource, method, append_allowed_methods=False): else: # If the node_type is referenced with an ObjectID (was not embedded on # request) query for if from the database and get the permissions - node_types_collection = app.data.driver.db['node_types'] - node_type = node_types_collection.find_one(resource['node_type']) + + # node_types_collection = app.data.driver.db['node_types'] + # node_type = node_types_collection.find_one(resource['node_type']) + + if type(resource['project']) is dict: + project = resource['project'] + else: + projects_collection = app.data.driver.db['projects'] + project = projects_collection.find_one(resource['project']) + node_type = next( + (item for item in project['node_types'] if item.get('name') \ + and item['name'] == resource['node_type']), None) computed_permissions = node_type['permissions'] else: computed_permissions = None @@ -449,6 +458,8 @@ app.on_fetched_item_node_types += before_returning_item_permissions app.on_fetched_resource_node_types += before_returning_resource_permissions app.on_replace_nodes += before_replacing_node app.on_insert_nodes += before_inserting_nodes +app.on_fetched_item_projects += before_returning_item_permissions +app.on_fetched_resource_projects += before_returning_resource_permissions def post_GET_user(request, payload): json_data = json.loads(payload.data) diff --git a/pillar/manage.py b/pillar/manage.py index efc62096..b02047b4 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -1,5 +1,6 @@ from __future__ import division import os +from bson.objectid import ObjectId from eve.methods.put import put_internal from eve.methods.post import post_internal from flask.ext.script import Manager @@ -55,6 +56,8 @@ def put_item(collection, item): internal_fields = ['_id', '_etag', '_updated', '_created'] for field in internal_fields: item.pop(field, None) + # print item + # print type(item_id) p = put_internal(collection, item, **{'_id': item_id}) if p[0]['_status'] == 'ERR': print p @@ -224,7 +227,7 @@ def add_parent_to_nodes(): """Find the parent of any node in the nodes collection""" import codecs import sys - from bson.objectid import ObjectId + UTF8Writer = codecs.getwriter('utf8') sys.stdout = UTF8Writer(sys.stdout) @@ -320,7 +323,7 @@ def remove_children_files(): @manager.command def make_project_public(project_id): """Convert every node of a project from pending to public""" - from bson.objectid import ObjectId + DRY_RUN = False nodes_collection = app.data.driver.db['nodes'] for n in nodes_collection.find({'project': ObjectId(project_id)}): @@ -405,11 +408,10 @@ def convert_assets_to_textures(project_id): if not DRY_RUN: p = post_internal('nodes', node) if p[0]['_status'] == 'ERR': - print p import pprint pprint.pprint(node) - from bson.objectid import ObjectId + nodes_collection = app.data.driver.db['nodes'] for n in nodes_collection.find({'project': ObjectId(project_id)}): @@ -494,5 +496,57 @@ def files_verify_project(): print i print "===" + +def replace_node_type(project, node_type_name, new_node_type): + """Update or create the specified node type. We rely on the fact that + node_types have a unique name in a project. + """ + + old_node_type = next( + (item for item in project['node_types'] if item.get('name') \ + and item['name'] == node_type_name), None) + if old_node_type: + for i, v in enumerate(project['node_types']): + if v['name'] == node_type_name: + project['node_types'][i] = new_node_type + else: + project['node_types'].append(new_node_type) + + +@manager.command +def project_upgrade_node_types(project_id): + projects_collection = app.data.driver.db['projects'] + project = projects_collection.find_one({'_id': ObjectId(project_id)}) + replace_node_type(project, 'group', node_type_group) + replace_node_type(project, 'asset', node_type_asset) + replace_node_type(project, 'storage', node_type_storage) + replace_node_type(project, 'comment', node_type_comment) + replace_node_type(project, 'blog', node_type_blog) + replace_node_type(project, 'post', node_type_post) + replace_node_type(project, 'texture', node_type_texture) + put_item('projects', project) + + +@manager.command +def test_put_item(node_id): + import pprint + nodes_collection = app.data.driver.db['nodes'] + node = nodes_collection.find_one(ObjectId(node_id)) + pprint.pprint(node) + put_item('nodes', node) + + +@manager.command +def test_post_internal(node_id): + import pprint + nodes_collection = app.data.driver.db['nodes'] + node = nodes_collection.find_one(ObjectId(node_id)) + internal_fields = ['_id', '_etag', '_updated', '_created'] + for field in internal_fields: + node.pop(field, None) + pprint.pprint(node) + print post_internal('nodes', node) + + if __name__ == '__main__': manager.run() diff --git a/pillar/manage/node_types/act.py b/pillar/manage/node_types/act.py index a50ac19a..d4346023 100644 --- a/pillar/manage/node_types/act.py +++ b/pillar/manage/node_types/act.py @@ -1,5 +1,5 @@ node_type_act = { 'name': 'act', 'description': 'Act node type', - 'parent': {} + 'parent': [] } diff --git a/pillar/manage/node_types/asset.py b/pillar/manage/node_types/asset.py index 50266de9..9aab4882 100644 --- a/pillar/manage/node_types/asset.py +++ b/pillar/manage/node_types/asset.py @@ -5,9 +5,7 @@ node_type_asset = { 'description': 'Basic Asset Type', # This data type does not have parent limitations (can be child # of any node). An empty parent declaration is required. - 'parent': { - "node_types": ["group",] - }, + 'parent': ['group',], 'dyn_schema': { 'status': { 'type': 'string', diff --git a/pillar/manage/node_types/blog.py b/pillar/manage/node_types/blog.py index 8718ed9c..45c82b97 100644 --- a/pillar/manage/node_types/blog.py +++ b/pillar/manage/node_types/blog.py @@ -17,9 +17,7 @@ node_type_blog = { 'categories': {}, 'template': {}, }, - 'parent': { - 'node_types': ['project',] - }, + 'parent': ['project',], 'permissions': { # 'groups': [{ # 'group': app.config['ADMIN_USER_GROUP'], diff --git a/pillar/manage/node_types/comment.py b/pillar/manage/node_types/comment.py index 673c8d63..bd25d245 100644 --- a/pillar/manage/node_types/comment.py +++ b/pillar/manage/node_types/comment.py @@ -60,9 +60,7 @@ node_type_comment = { 'confidence': {}, 'is_reply': {} }, - 'parent': { - 'node_types': ['asset', 'comment'] - }, + 'parent': ['asset', 'comment'], 'permissions': { # 'groups': [{ # 'group': app.config['ADMIN_USER_GROUP'], diff --git a/pillar/manage/node_types/group.py b/pillar/manage/node_types/group.py index 4eaf7591..5fb7a4a3 100644 --- a/pillar/manage/node_types/group.py +++ b/pillar/manage/node_types/group.py @@ -1,9 +1,7 @@ node_type_group = { 'name': 'group', - 'description': 'Generic group node type', - 'parent': { - 'node_types': ['group', 'project'] - }, + 'description': 'Generic group node type edited', + 'parent': ['group', 'project'], 'dyn_schema': { # Used for sorting within the context of a group 'order': { diff --git a/pillar/manage/node_types/group_texture.py b/pillar/manage/node_types/group_texture.py index 81635851..eb4affff 100644 --- a/pillar/manage/node_types/group_texture.py +++ b/pillar/manage/node_types/group_texture.py @@ -1,9 +1,7 @@ node_type_group_texture = { 'name': 'group_texture', 'description': 'Group for texture node type', - 'parent': { - 'node_types': ['group_texture', 'project'] - }, + 'parent': ['group_texture', 'project'], 'dyn_schema': { # Used for sorting within the context of a group 'order': { diff --git a/pillar/manage/node_types/post.py b/pillar/manage/node_types/post.py index 82c08040..916e9242 100644 --- a/pillar/manage/node_types/post.py +++ b/pillar/manage/node_types/post.py @@ -55,9 +55,7 @@ node_type_post = { 'url': {}, 'attachments': {'visible': False}, }, - 'parent': { - 'node_types': ['blog',] - }, + 'parent': ['blog',], 'permissions': { # 'groups': [{ # 'group': app.config['ADMIN_USER_GROUP'], diff --git a/pillar/manage/node_types/scene.py b/pillar/manage/node_types/scene.py index ca710ed3..99778756 100644 --- a/pillar/manage/node_types/scene.py +++ b/pillar/manage/node_types/scene.py @@ -1,7 +1,5 @@ node_type_scene = { 'name': 'scene', 'description': 'Scene node type', - 'parent': { - "node_types": ["act"] - } + 'parent': ['act'], } diff --git a/pillar/manage/node_types/shot.py b/pillar/manage/node_types/shot.py index b168a8c3..99b54606 100644 --- a/pillar/manage/node_types/shot.py +++ b/pillar/manage/node_types/shot.py @@ -41,7 +41,5 @@ node_type_shot = { 'notes': {}, 'shot_group': {} }, - 'parent': { - 'node_types': ['scene'] - } + 'parent': ['scene'] } diff --git a/pillar/manage/node_types/storage.py b/pillar/manage/node_types/storage.py index 8e273a2a..bc573529 100644 --- a/pillar/manage/node_types/storage.py +++ b/pillar/manage/node_types/storage.py @@ -26,9 +26,7 @@ node_type_storage = { 'project': {}, 'backend': {} }, - 'parent': { - "node_types": ["group", "project"] - }, + 'parent': ['group', 'project'], 'permissions': { # 'groups': [{ # 'group': app.config['ADMIN_USER_GROUP'], diff --git a/pillar/manage/node_types/task.py b/pillar/manage/node_types/task.py index 2b83fdd4..5fdb3500 100644 --- a/pillar/manage/node_types/task.py +++ b/pillar/manage/node_types/task.py @@ -103,7 +103,5 @@ node_type_task = { 'is_open': {}, 'is_processing': {}, }, - 'parent': { - 'node_types': ['shot'], - } + 'parent': ['shot'] } diff --git a/pillar/manage/node_types/texture.py b/pillar/manage/node_types/texture.py index 446e55ef..5c7df999 100644 --- a/pillar/manage/node_types/texture.py +++ b/pillar/manage/node_types/texture.py @@ -5,9 +5,7 @@ node_type_texture = { 'description': 'Image Texture', # This data type does not have parent limitations (can be child # of any node). An empty parent declaration is required. - 'parent': { - "node_types": ["group",] - }, + 'parent': ['group',], 'dyn_schema': { 'status': { 'type': 'string', diff --git a/pillar/settings.py b/pillar/settings.py index 1ed8cf84..f41dcc7a 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -11,6 +11,15 @@ ITEM_METHODS = ['GET', 'PUT', 'DELETE', 'PATCH'] PAGINATION_LIMIT = 25 +_file_embedded_schema = { + 'type': 'objectid', + 'data_relation': { + 'resource': 'files', + 'field': '_id', + 'embeddable': True + } +} + users_schema = { 'full_name': { @@ -258,7 +267,7 @@ nodes_schema = { 'project': { 'type': 'objectid', 'data_relation': { - 'resource': 'nodes', + 'resource': 'projects', 'field': '_id', 'embeddable': True }, @@ -273,13 +282,8 @@ nodes_schema = { }, }, 'node_type': { - 'type': 'objectid', - 'required': True, - 'data_relation': { - 'resource': 'node_types', - 'field': '_id', - 'embeddable': True - }, + 'type': 'string', + 'required': True }, 'properties': { 'type' : 'dict', @@ -502,6 +506,151 @@ groups_schema = { } } +projects_schema = { + 'name': { + 'type': 'string', + 'minlength': 1, + 'maxlength': 128, + 'required': True, + }, + 'description': { + 'type': 'string', + }, + # Short summary for the project + 'summary': { + 'type': 'string', + 'maxlength': 128 + }, + # Logo + 'picture_square': _file_embedded_schema, + # Header + 'picture_header': _file_embedded_schema, + 'user': { + 'type': 'objectid', + 'required': True, + 'data_relation': { + 'resource': 'users', + 'field': '_id', + 'embeddable': True + }, + }, + 'category': { + 'type': 'string', + 'allowed': [ + 'training', + 'film', + 'assets', + 'software', + 'game' + ], + 'required': True, + }, + 'is_private': { + 'type': 'boolean' + }, + 'url': { + 'type': 'string' + }, + 'organization': { + 'type': 'objectid', + 'nullable': True, + 'data_relation': { + 'resource': 'organizations', + 'field': '_id', + 'embeddable': True + }, + }, + 'owners': { + 'type': 'dict', + 'schema': { + 'users': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + 'groups': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + 'data_relation': { + 'resource': 'groups', + 'field': '_id', + 'embeddable': True + } + } + } + } + }, + 'status': { + 'type': 'string', + 'allowed': [ + 'published', + 'pending', + 'deleted' + ], + }, + # Latest nodes being edited + 'nodes_latest': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + # Featured nodes, manually added + 'nodes_featured': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + # Latest blog posts, manually added + 'nodes_blog': { + 'type': 'list', + 'schema': { + 'type': 'objectid', + } + }, + # Where Node type schemas for every projects are defined + 'node_types': { + 'type': 'list', + 'schema': { + 'type': 'dict', + 'schema': { + # URL is the way we identify a node_type when calling it via + # the helper methods in the Project API. + 'url': {'type': 'string'}, + 'name': {'type': 'string'}, + 'description': {'type': 'string'}, + # Allowed parents for the node_type + 'parent': { + 'type': 'list', + 'schema': { + 'type': 'string' + } + }, + 'dyn_schema': { + 'type': 'dict', + 'allow_unknown': True + }, + 'form_schema': { + 'type': 'dict', + 'allow_unknown': True + }, + 'permissions': { + 'type': 'dict', + 'schema': permissions_embedded_schema + } + }, + + } + }, + 'permissions': { + 'type': 'dict', + 'schema': permissions_embedded_schema + } +} + nodes = { 'schema': nodes_schema, 'public_methods': ['GET'], @@ -559,6 +708,12 @@ organizations = { 'public_methods': ['GET'] } +projects = { + 'schema': projects_schema, + 'public_item_methods': ['GET'], + 'public_methods': ['GET'] +} + DOMAIN = { 'users': users, 'nodes': nodes, @@ -566,7 +721,8 @@ DOMAIN = { 'tokens': tokens, 'files': files, 'groups': groups, - 'organizations': organizations + 'organizations': organizations, + 'projects': projects }