Introducing Projects

We are now using a more document-based approach to define projects. In
the new projects collection we store the definition of a project and
embed the node_types. This allows for custom node_types for every
single project. This change has a certain impact on the custom
validators, as well as the permission computation.
Further, Cerberus 0.9.1 is required in order to properly support the
allow_unknown statements in the projects_schema definition.

pillar/application/__init__.py

@@ -195,16 +195,16 @@ class ValidateCustomFields(Validator):
         return properties
 
     def _validate_valid_properties(self, valid_properties, field, value):
-        node_types = app.data.driver.db['node_types']
+        projects_collection = app.data.driver.db['projects']
-        lookup = {}
+        lookup = {'_id': ObjectId(self.document['project'])}
-        lookup['_id'] = ObjectId(self.document['node_type'])
+        project = projects_collection.find_one(lookup)
-        node_type = node_types.find_one(lookup)
+        node_type = next(
-
+            (item for item in project['node_types'] if item.get('name') \
+                and item['name'] == self.document['node_type']), None)
         try:
             value = self.convert_properties(value, node_type['dyn_schema'])
         except Exception, e:
             print ("Error converting: {0}".format(e))
-        #print (value)
 
         v = Validator(node_type['dyn_schema'])
         val = v.validate(value)
@@ -309,7 +309,6 @@ def check_permissions(resource, method, append_allowed_methods=False):
         resource_permissions = resource['permissions']
     else:
         resource_permissions = None
-
     if 'node_type' in resource:
         if type(resource['node_type']) is dict:
             # If the node_type is embedded in the document, extract permissions
@@ -318,8 +317,18 @@ def check_permissions(resource, method, append_allowed_methods=False):
         else:
             # If the node_type is referenced with an ObjectID (was not embedded on
             # request) query for if from the database and get the permissions
-            node_types_collection = app.data.driver.db['node_types']
+
-            node_type = node_types_collection.find_one(resource['node_type'])
+            # node_types_collection = app.data.driver.db['node_types']
+            # node_type = node_types_collection.find_one(resource['node_type'])
+
+            if type(resource['project']) is dict:
+                project = resource['project']
+            else:
+                projects_collection = app.data.driver.db['projects']
+                project = projects_collection.find_one(resource['project'])
+            node_type = next(
+                (item for item in project['node_types'] if item.get('name') \
+                    and item['name'] == resource['node_type']), None)
             computed_permissions = node_type['permissions']
     else:
         computed_permissions = None
@@ -449,6 +458,8 @@ app.on_fetched_item_node_types += before_returning_item_permissions
 app.on_fetched_resource_node_types += before_returning_resource_permissions
 app.on_replace_nodes += before_replacing_node
 app.on_insert_nodes += before_inserting_nodes
+app.on_fetched_item_projects += before_returning_item_permissions
+app.on_fetched_resource_projects += before_returning_resource_permissions
 
 def post_GET_user(request, payload):
     json_data = json.loads(payload.data)

pillar/manage.py

@@ -1,5 +1,6 @@
 from __future__ import division
 import os
+from bson.objectid import ObjectId
 from eve.methods.put import put_internal
 from eve.methods.post import post_internal
 from flask.ext.script import Manager
@@ -55,6 +56,8 @@ def put_item(collection, item):
     internal_fields = ['_id', '_etag', '_updated', '_created']
     for field in internal_fields:
         item.pop(field, None)
+    # print item
+    # print type(item_id)
     p = put_internal(collection, item, **{'_id': item_id})
     if p[0]['_status'] == 'ERR':
         print p
@@ -224,7 +227,7 @@ def add_parent_to_nodes():
     """Find the parent of any node in the nodes collection"""
     import codecs
     import sys
-    from bson.objectid import ObjectId
+
     UTF8Writer = codecs.getwriter('utf8')
     sys.stdout = UTF8Writer(sys.stdout)
 
@@ -320,7 +323,7 @@ def remove_children_files():
 @manager.command
 def make_project_public(project_id):
     """Convert every node of a project from pending to public"""
-    from bson.objectid import ObjectId
+
     DRY_RUN = False
     nodes_collection = app.data.driver.db['nodes']
     for n in nodes_collection.find({'project': ObjectId(project_id)}):
@@ -405,11 +408,10 @@ def convert_assets_to_textures(project_id):
             if not DRY_RUN:
                 p = post_internal('nodes', node)
                 if p[0]['_status'] == 'ERR':
-                    print p
                     import pprint
                     pprint.pprint(node)
 
-    from bson.objectid import ObjectId
+
     nodes_collection = app.data.driver.db['nodes']
 
     for n in nodes_collection.find({'project': ObjectId(project_id)}):
@@ -494,5 +496,57 @@ def files_verify_project():
             print i
         print "==="
 
+
+def replace_node_type(project, node_type_name, new_node_type):
+    """Update or create the specified node type. We rely on the fact that
+    node_types have a unique name in a project.
+    """
+
+    old_node_type = next(
+        (item for item in project['node_types'] if item.get('name') \
+            and item['name'] == node_type_name), None)
+    if old_node_type:
+        for i, v in enumerate(project['node_types']):
+            if v['name'] == node_type_name:
+                project['node_types'][i] = new_node_type
+    else:
+        project['node_types'].append(new_node_type)
+
+
+@manager.command
+def project_upgrade_node_types(project_id):
+    projects_collection = app.data.driver.db['projects']
+    project = projects_collection.find_one({'_id': ObjectId(project_id)})
+    replace_node_type(project, 'group', node_type_group)
+    replace_node_type(project, 'asset', node_type_asset)
+    replace_node_type(project, 'storage', node_type_storage)
+    replace_node_type(project, 'comment', node_type_comment)
+    replace_node_type(project, 'blog', node_type_blog)
+    replace_node_type(project, 'post', node_type_post)
+    replace_node_type(project, 'texture', node_type_texture)
+    put_item('projects', project)
+
+
+@manager.command
+def test_put_item(node_id):
+    import pprint
+    nodes_collection = app.data.driver.db['nodes']
+    node = nodes_collection.find_one(ObjectId(node_id))
+    pprint.pprint(node)
+    put_item('nodes', node)
+
+
+@manager.command
+def test_post_internal(node_id):
+    import pprint
+    nodes_collection = app.data.driver.db['nodes']
+    node = nodes_collection.find_one(ObjectId(node_id))
+    internal_fields = ['_id', '_etag', '_updated', '_created']
+    for field in internal_fields:
+        node.pop(field, None)
+    pprint.pprint(node)
+    print post_internal('nodes', node)
+
+
 if __name__ == '__main__':
     manager.run()

pillar/manage/node_types/act.py

@@ -1,5 +1,5 @@
 node_type_act = {
     'name': 'act',
     'description': 'Act node type',
-    'parent': {}
+    'parent': []
 }

pillar/manage/node_types/asset.py

@@ -5,9 +5,7 @@ node_type_asset = {
     'description': 'Basic Asset Type',
     # This data type does not have parent limitations (can be child
     # of any node). An empty parent declaration is required.
-    'parent': {
+    'parent': ['group',],
-        "node_types": ["group",]
-    },
     'dyn_schema': {
         'status': {
             'type': 'string',

pillar/manage/node_types/blog.py

@@ -17,9 +17,7 @@ node_type_blog = {
         'categories': {},
         'template': {},
     },
-    'parent': {
+    'parent': ['project',],
-        'node_types': ['project',]
-    },
     'permissions': {
         # 'groups': [{
         #     'group': app.config['ADMIN_USER_GROUP'],

pillar/manage/node_types/comment.py

@@ -60,9 +60,7 @@ node_type_comment = {
         'confidence': {},
         'is_reply': {}
     },
-    'parent': {
+    'parent': ['asset', 'comment'],
-        'node_types': ['asset', 'comment']
-    },
     'permissions': {
         # 'groups': [{
         #     'group': app.config['ADMIN_USER_GROUP'],

pillar/manage/node_types/group.py

@@ -1,9 +1,7 @@
 node_type_group = {
     'name': 'group',
-    'description': 'Generic group node type',
+    'description': 'Generic group node type edited',
-    'parent': {
+    'parent': ['group', 'project'],
-        'node_types': ['group', 'project']
-    },
     'dyn_schema': {
         # Used for sorting within the context of a group
         'order': {

pillar/manage/node_types/group_texture.py

@@ -1,9 +1,7 @@
 node_type_group_texture = {
     'name': 'group_texture',
     'description': 'Group for texture node type',
-    'parent': {
+    'parent': ['group_texture', 'project'],
-        'node_types': ['group_texture', 'project']
-    },
     'dyn_schema': {
         # Used for sorting within the context of a group
         'order': {

pillar/manage/node_types/post.py

@@ -55,9 +55,7 @@ node_type_post = {
         'url': {},
         'attachments': {'visible': False},
     },
-    'parent': {
+    'parent': ['blog',],
-        'node_types': ['blog',]
-    },
     'permissions': {
         # 'groups': [{
         #     'group': app.config['ADMIN_USER_GROUP'],

pillar/manage/node_types/scene.py

@@ -1,7 +1,5 @@
 node_type_scene = {
     'name': 'scene',
     'description': 'Scene node type',
-    'parent': {
+    'parent': ['act'],
-        "node_types": ["act"]
-    }
 }

pillar/manage/node_types/shot.py

@@ -41,7 +41,5 @@ node_type_shot = {
         'notes': {},
         'shot_group': {}
     },
-    'parent': {
+    'parent': ['scene']
-        'node_types': ['scene']
-    }
 }

pillar/manage/node_types/storage.py

@@ -26,9 +26,7 @@ node_type_storage = {
         'project': {},
         'backend': {}
     },
-    'parent': {
+    'parent': ['group', 'project'],
-        "node_types": ["group", "project"]
-    },
     'permissions': {
         # 'groups': [{
         #     'group': app.config['ADMIN_USER_GROUP'],

pillar/manage/node_types/task.py

@@ -103,7 +103,5 @@ node_type_task = {
         'is_open': {},
         'is_processing': {},
     },
-    'parent': {
+    'parent': ['shot']
-        'node_types': ['shot'],
-    }
 }

pillar/manage/node_types/texture.py

@@ -5,9 +5,7 @@ node_type_texture = {
     'description': 'Image Texture',
     # This data type does not have parent limitations (can be child
     # of any node). An empty parent declaration is required.
-    'parent': {
+    'parent': ['group',],
-        "node_types": ["group",]
-    },
     'dyn_schema': {
         'status': {
             'type': 'string',

pillar/settings.py

@@ -11,6 +11,15 @@ ITEM_METHODS = ['GET', 'PUT', 'DELETE', 'PATCH']
 
 PAGINATION_LIMIT = 25
 
+_file_embedded_schema = {
+    'type': 'objectid',
+    'data_relation': {
+        'resource': 'files',
+        'field': '_id',
+        'embeddable': True
+    }
+}
+
 
 users_schema = {
     'full_name': {
@@ -258,7 +267,7 @@ nodes_schema = {
     'project': {
         'type': 'objectid',
          'data_relation': {
-            'resource': 'nodes',
+            'resource': 'projects',
             'field': '_id',
             'embeddable': True
          },
@@ -273,13 +282,8 @@ nodes_schema = {
         },
     },
     'node_type': {
-        'type': 'objectid',
+        'type': 'string',
-        'required': True,
+        'required': True
-        'data_relation': {
-            'resource': 'node_types',
-            'field': '_id',
-            'embeddable': True
-        },
     },
     'properties': {
         'type' : 'dict',
@@ -502,6 +506,151 @@ groups_schema = {
     }
 }
 
+projects_schema = {
+    'name': {
+        'type': 'string',
+        'minlength': 1,
+        'maxlength': 128,
+        'required': True,
+    },
+    'description': {
+        'type': 'string',
+    },
+    # Short summary for the project
+    'summary': {
+        'type': 'string',
+        'maxlength': 128
+    },
+    # Logo
+    'picture_square': _file_embedded_schema,
+    # Header
+    'picture_header': _file_embedded_schema,
+    'user': {
+        'type': 'objectid',
+        'required': True,
+        'data_relation': {
+            'resource': 'users',
+            'field': '_id',
+            'embeddable': True
+        },
+    },
+    'category': {
+        'type': 'string',
+        'allowed': [
+            'training',
+            'film',
+            'assets',
+            'software',
+            'game'
+        ],
+        'required': True,
+    },
+    'is_private': {
+        'type': 'boolean'
+    },
+    'url': {
+        'type': 'string'
+    },
+    'organization': {
+        'type': 'objectid',
+        'nullable': True,
+        'data_relation': {
+           'resource': 'organizations',
+           'field': '_id',
+           'embeddable': True
+        },
+    },
+    'owners': {
+        'type': 'dict',
+        'schema': {
+            'users': {
+                'type': 'list',
+                'schema': {
+                    'type': 'objectid',
+                }
+            },
+            'groups': {
+                'type': 'list',
+                'schema': {
+                    'type': 'objectid',
+                    'data_relation': {
+                        'resource': 'groups',
+                        'field': '_id',
+                        'embeddable': True
+                    }
+                }
+            }
+        }
+    },
+    'status': {
+        'type': 'string',
+        'allowed': [
+            'published',
+            'pending',
+            'deleted'
+        ],
+    },
+    # Latest nodes being edited
+    'nodes_latest': {
+        'type': 'list',
+        'schema': {
+            'type': 'objectid',
+        }
+    },
+    # Featured nodes, manually added
+    'nodes_featured': {
+        'type': 'list',
+        'schema': {
+            'type': 'objectid',
+        }
+    },
+    # Latest blog posts, manually added
+    'nodes_blog': {
+        'type': 'list',
+        'schema': {
+            'type': 'objectid',
+        }
+    },
+    # Where Node type schemas for every projects are defined
+    'node_types': {
+        'type': 'list',
+        'schema': {
+            'type': 'dict',
+            'schema': {
+                # URL is the way we identify a node_type when calling it via
+                # the helper methods in the Project API.
+                'url': {'type': 'string'},
+                'name': {'type': 'string'},
+                'description': {'type': 'string'},
+                # Allowed parents for the node_type
+                'parent': {
+                    'type': 'list',
+                    'schema': {
+                        'type': 'string'
+                    }
+                },
+                'dyn_schema': {
+                    'type': 'dict',
+                    'allow_unknown': True
+                },
+                'form_schema': {
+                    'type': 'dict',
+                    'allow_unknown': True
+                },
+                'permissions': {
+                    'type': 'dict',
+                    'schema': permissions_embedded_schema
+                }
+            },
+
+        }
+    },
+    'permissions': {
+        'type': 'dict',
+        'schema': permissions_embedded_schema
+    }
+}
+
 nodes = {
     'schema': nodes_schema,
     'public_methods': ['GET'],
@@ -559,6 +708,12 @@ organizations = {
     'public_methods': ['GET']
 }
 
+projects = {
+    'schema': projects_schema,
+    'public_item_methods': ['GET'],
+    'public_methods': ['GET']
+}
+
 DOMAIN = {
     'users': users,
     'nodes': nodes,
@@ -566,7 +721,8 @@ DOMAIN = {
     'tokens': tokens,
     'files': files,
     'groups': groups,
-    'organizations': organizations
+    'organizations': organizations,
+    'projects': projects
 }