New data structure for attachments.

pillar/api/custom_field_validation.py

@@ -21,8 +21,13 @@ class ValidateCustomFields(Validator):
             prop_type = schema_prop['type']
 
             if prop_type == 'dict':
-                properties[prop] = self.convert_properties(
-                    properties[prop], schema_prop['schema'])
+                try:
+                    dict_valueschema = schema_prop['schema']
+                except KeyError:
+                    # TODO: will be renamed to 'keyschema' in Cerberus 1.0
+                    dict_valueschema = schema_prop['valueschema']
+                properties[prop] = self.convert_properties(properties[prop], dict_valueschema)
+
             elif prop_type == 'list':
                 if properties[prop] in ['', '[]']:
                     properties[prop] = []

pillar/api/node_types/__init__.py

@@ -17,7 +17,10 @@ _attachments_embedded_schema = {
     'valueschema': {
         'type': 'dict',
         'schema': {
-            'oid': 'objectid',
+            'oid': {
+                'type': 'objectid',
+                'required': True,
+            },
             'collection': {
                 'type': 'string',
                 'allowed': ['files'],

pillar/api/projects/utils.py

@@ -90,3 +90,10 @@ def create_new_project(project_name, user_id, overrides):
     log.info('Created project %s for user %s', project['_id'], user_id)
 
     return project
+
+
+def get_node_type(project, node_type_name):
+    """Returns the named node type, or None if it doesn't exist."""
+
+    return next((nt for nt in project['node_types']
+                 if nt['name'] == node_type_name), None)

pillar/cli.py

@@ -627,3 +627,96 @@ def remarkdown_comments():
     log.info('identical: %i', identical)
     log.info('skipped  : %i', skipped)
     log.info('errors   : %i', errors)
+
+
+@manager.command
+@manager.option('-p', '--project', dest='proj_url', nargs='?',
+                help='Project URL')
+@manager.option('-a', '--all', dest='all_projects', action='store_true', default=False,
+                help='Replace on all projects.')
+def upgrade_attachment_schema(proj_url=None, all_projects=False):
+    """Replaces the project's attachments with the new schema.
+
+    Updates both the schema definition and the nodes with attachments (asset, page, post).
+    """
+
+    if bool(proj_url) == all_projects:
+        log.error('Use either --project or --all.')
+        return 1
+
+    from pillar.api.utils.authentication import force_cli_user
+    force_cli_user()
+
+    from pillar.api.node_types.asset import node_type_asset
+    from pillar.api.node_types.page import node_type_page
+    from pillar.api.node_types.post import node_type_post
+    from pillar.api.node_types import _attachments_embedded_schema
+    from pillar.api.utils import remove_private_keys
+
+    # Node types that support attachments
+    node_types = (node_type_asset, node_type_page, node_type_post)
+    node_type_names = {nt['name'] for nt in node_types}
+
+    db = current_app.db()
+    projects_coll = db['projects']
+    nodes_coll = db['nodes']
+
+    def handle_project(project):
+        log.info('Handling project %s', project['url'])
+
+        replace_schemas(project)
+        replace_attachments(project)
+
+    def replace_schemas(project):
+        for proj_nt in project['node_types']:
+            nt_name = proj_nt['name']
+            if nt_name not in node_type_names:
+                log.info('   - skipping node type "%s"', nt_name)
+                continue
+
+            log.info('   - replacing attachment schema on node type "%s"', nt_name)
+            proj_nt['dyn_schema']['attachments'] = copy.deepcopy(_attachments_embedded_schema)
+
+        # Use Eve to PUT, so we have schema checking.
+        db_proj = remove_private_keys(project)
+        r, _, _, status = put_internal('projects', db_proj, _id=project['_id'])
+        if status != 200:
+            log.error('Error %i storing altered project %s: %s', status, project['_id'], r)
+            return 4
+        log.info('Project saved succesfully.')
+
+    def replace_attachments(project):
+        log.info('Upgrading nodes for project %s', project['url'])
+        nodes = nodes_coll.find({
+            'project': project['_id'],
+            'node_type': {'$in': list(node_type_names)},
+            'properties.attachments.0': {'$exists': True},
+        })
+        for node in nodes:
+            log.info('    - Updating schema on node %s (%s)', node['_id'], node.get('name'))
+            new_atts = {}
+
+            for field_info in node[u'properties'][u'attachments']:
+                for attachment in field_info.get('files', []):
+                    new_atts[attachment[u'slug']] = {u'oid': attachment[u'file']}
+
+            node[u'properties'][u'attachments'] = new_atts
+
+            # Use Eve to PUT, so we have schema checking.
+            db_node = remove_private_keys(node)
+            r, _, _, status = put_internal('nodes', db_node, _id=node['_id'])
+            if status != 200:
+                log.error('Error %i storing altered node %s: %s', status, node['_id'], r)
+                return
+
+    if all_projects:
+        for proj in projects_coll.find():
+            handle_project(proj)
+        return
+
+    proj = projects_coll.find_one({'url': proj_url})
+    if not proj:
+        log.error('Project url=%s not found', proj_url)
+        return 3
+
+    handle_project(proj)

pillar/tests/__init__.py

@@ -109,7 +109,11 @@ class AbstractPillarTest(TestMinimal):
             del sys.modules[modname]
 
     def ensure_file_exists(self, file_overrides=None):
-        self.ensure_project_exists()
+        if file_overrides and file_overrides.get('project'):
+            self.ensure_project_exists({'_id': file_overrides['project']})
+        else:
+            self.ensure_project_exists()
+
         with self.app.test_request_context():
             files_collection = self.app.data.driver.db['files']
             assert isinstance(files_collection, pymongo.collection.Collection)
@@ -222,7 +226,7 @@ class AbstractPillarTest(TestMinimal):
 
         return token_data
 
-    def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )):
+    def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)):
         """Creates a project and a user that's member of the project's admin group.
 
         :returns: (project_id, user_id)
@@ -233,7 +237,7 @@ class AbstractPillarTest(TestMinimal):
 
         return project_id, user_id
 
-    def create_project_admin(self, proj, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )):
+    def create_project_admin(self, proj, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)):
         """Creates a user that's member of the project's admin group.
 
         :param proj: project document, or at least a dict with permissions in it.
@@ -247,6 +251,14 @@ class AbstractPillarTest(TestMinimal):
 
         return user_id
 
+    def create_node(self, node_doc):
+        """Creates a node, returning its ObjectId. """
+
+        with self.app.test_request_context():
+            nodes_coll = self.app.data.driver.db['nodes']
+            result = nodes_coll.insert_one(node_doc)
+        return result.inserted_id
+
     def badger(self, user_email, roles, action, srv_token=None):
         """Creates a service account, and uses it to grant or revoke a role to the user.