From 0929a80f2bd874f2181477e04ab4377b9454c7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 25 Oct 2016 15:24:06 +0200 Subject: [PATCH] New data structure for attachments. --- pillar/api/custom_field_validation.py | 9 +- pillar/api/node_types/__init__.py | 5 +- pillar/api/projects/utils.py | 7 + pillar/cli.py | 93 ++++++ pillar/tests/__init__.py | 18 +- tests/test_api/test_bcloud_home_project.py | 5 - tests/test_api/test_cli.py | 331 +++++++++++++++++++-- 7 files changed, 436 insertions(+), 32 deletions(-) diff --git a/pillar/api/custom_field_validation.py b/pillar/api/custom_field_validation.py index d9fc547e..8bb8b71b 100644 --- a/pillar/api/custom_field_validation.py +++ b/pillar/api/custom_field_validation.py @@ -21,8 +21,13 @@ class ValidateCustomFields(Validator): prop_type = schema_prop['type'] if prop_type == 'dict': - properties[prop] = self.convert_properties( - properties[prop], schema_prop['schema']) + try: + dict_valueschema = schema_prop['schema'] + except KeyError: + # TODO: will be renamed to 'keyschema' in Cerberus 1.0 + dict_valueschema = schema_prop['valueschema'] + properties[prop] = self.convert_properties(properties[prop], dict_valueschema) + elif prop_type == 'list': if properties[prop] in ['', '[]']: properties[prop] = [] diff --git a/pillar/api/node_types/__init__.py b/pillar/api/node_types/__init__.py index 56ab6381..ddb8aede 100644 --- a/pillar/api/node_types/__init__.py +++ b/pillar/api/node_types/__init__.py @@ -17,7 +17,10 @@ _attachments_embedded_schema = { 'valueschema': { 'type': 'dict', 'schema': { - 'oid': 'objectid', + 'oid': { + 'type': 'objectid', + 'required': True, + }, 'collection': { 'type': 'string', 'allowed': ['files'], diff --git a/pillar/api/projects/utils.py b/pillar/api/projects/utils.py index a7c6df3e..d38fc21f 100644 --- a/pillar/api/projects/utils.py +++ b/pillar/api/projects/utils.py @@ -90,3 +90,10 @@ def create_new_project(project_name, user_id, overrides): log.info('Created project %s for user %s', project['_id'], user_id) return project + + +def get_node_type(project, node_type_name): + """Returns the named node type, or None if it doesn't exist.""" + + return next((nt for nt in project['node_types'] + if nt['name'] == node_type_name), None) diff --git a/pillar/cli.py b/pillar/cli.py index 41c1a8cd..cdf1f673 100644 --- a/pillar/cli.py +++ b/pillar/cli.py @@ -627,3 +627,96 @@ def remarkdown_comments(): log.info('identical: %i', identical) log.info('skipped : %i', skipped) log.info('errors : %i', errors) + + +@manager.command +@manager.option('-p', '--project', dest='proj_url', nargs='?', + help='Project URL') +@manager.option('-a', '--all', dest='all_projects', action='store_true', default=False, + help='Replace on all projects.') +def upgrade_attachment_schema(proj_url=None, all_projects=False): + """Replaces the project's attachments with the new schema. + + Updates both the schema definition and the nodes with attachments (asset, page, post). + """ + + if bool(proj_url) == all_projects: + log.error('Use either --project or --all.') + return 1 + + from pillar.api.utils.authentication import force_cli_user + force_cli_user() + + from pillar.api.node_types.asset import node_type_asset + from pillar.api.node_types.page import node_type_page + from pillar.api.node_types.post import node_type_post + from pillar.api.node_types import _attachments_embedded_schema + from pillar.api.utils import remove_private_keys + + # Node types that support attachments + node_types = (node_type_asset, node_type_page, node_type_post) + node_type_names = {nt['name'] for nt in node_types} + + db = current_app.db() + projects_coll = db['projects'] + nodes_coll = db['nodes'] + + def handle_project(project): + log.info('Handling project %s', project['url']) + + replace_schemas(project) + replace_attachments(project) + + def replace_schemas(project): + for proj_nt in project['node_types']: + nt_name = proj_nt['name'] + if nt_name not in node_type_names: + log.info(' - skipping node type "%s"', nt_name) + continue + + log.info(' - replacing attachment schema on node type "%s"', nt_name) + proj_nt['dyn_schema']['attachments'] = copy.deepcopy(_attachments_embedded_schema) + + # Use Eve to PUT, so we have schema checking. + db_proj = remove_private_keys(project) + r, _, _, status = put_internal('projects', db_proj, _id=project['_id']) + if status != 200: + log.error('Error %i storing altered project %s: %s', status, project['_id'], r) + return 4 + log.info('Project saved succesfully.') + + def replace_attachments(project): + log.info('Upgrading nodes for project %s', project['url']) + nodes = nodes_coll.find({ + 'project': project['_id'], + 'node_type': {'$in': list(node_type_names)}, + 'properties.attachments.0': {'$exists': True}, + }) + for node in nodes: + log.info(' - Updating schema on node %s (%s)', node['_id'], node.get('name')) + new_atts = {} + + for field_info in node[u'properties'][u'attachments']: + for attachment in field_info.get('files', []): + new_atts[attachment[u'slug']] = {u'oid': attachment[u'file']} + + node[u'properties'][u'attachments'] = new_atts + + # Use Eve to PUT, so we have schema checking. + db_node = remove_private_keys(node) + r, _, _, status = put_internal('nodes', db_node, _id=node['_id']) + if status != 200: + log.error('Error %i storing altered node %s: %s', status, node['_id'], r) + return + + if all_projects: + for proj in projects_coll.find(): + handle_project(proj) + return + + proj = projects_coll.find_one({'url': proj_url}) + if not proj: + log.error('Project url=%s not found', proj_url) + return 3 + + handle_project(proj) diff --git a/pillar/tests/__init__.py b/pillar/tests/__init__.py index 707be923..0d5081ea 100644 --- a/pillar/tests/__init__.py +++ b/pillar/tests/__init__.py @@ -109,7 +109,11 @@ class AbstractPillarTest(TestMinimal): del sys.modules[modname] def ensure_file_exists(self, file_overrides=None): - self.ensure_project_exists() + if file_overrides and file_overrides.get('project'): + self.ensure_project_exists({'_id': file_overrides['project']}) + else: + self.ensure_project_exists() + with self.app.test_request_context(): files_collection = self.app.data.driver.db['files'] assert isinstance(files_collection, pymongo.collection.Collection) @@ -222,7 +226,7 @@ class AbstractPillarTest(TestMinimal): return token_data - def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )): + def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)): """Creates a project and a user that's member of the project's admin group. :returns: (project_id, user_id) @@ -233,7 +237,7 @@ class AbstractPillarTest(TestMinimal): return project_id, user_id - def create_project_admin(self, proj, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )): + def create_project_admin(self, proj, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)): """Creates a user that's member of the project's admin group. :param proj: project document, or at least a dict with permissions in it. @@ -247,6 +251,14 @@ class AbstractPillarTest(TestMinimal): return user_id + def create_node(self, node_doc): + """Creates a node, returning its ObjectId. """ + + with self.app.test_request_context(): + nodes_coll = self.app.data.driver.db['nodes'] + result = nodes_coll.insert_one(node_doc) + return result.inserted_id + def badger(self, user_email, roles, action, srv_token=None): """Creates a service account, and uses it to grant or revoke a role to the user. diff --git a/tests/test_api/test_bcloud_home_project.py b/tests/test_api/test_bcloud_home_project.py index d0d74c8a..bb89ec94 100644 --- a/tests/test_api/test_bcloud_home_project.py +++ b/tests/test_api/test_bcloud_home_project.py @@ -404,11 +404,6 @@ class HomeProjectUserChangedRoleTest(AbstractPillarTest): class TextureLibraryTest(AbstractHomeProjectTest): - def create_node(self, node_doc): - with self.app.test_request_context(): - nodes_coll = self.app.data.driver.db['nodes'] - result = nodes_coll.insert_one(node_doc) - return result.inserted_id def setUp(self, **kwargs): AbstractHomeProjectTest.setUp(self, **kwargs) diff --git a/tests/test_api/test_cli.py b/tests/test_api/test_cli.py index b1b83b00..c2adc2fd 100644 --- a/tests/test_api/test_cli.py +++ b/tests/test_api/test_cli.py @@ -1,27 +1,279 @@ +# -*- encoding: utf-8 -*- + from __future__ import absolute_import +import datetime + +from bson import tz_util, ObjectId from pillar.tests import AbstractPillarTest from pillar.tests import common_test_data as ctd +from pillar.api.projects.utils import get_node_type + +EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID = ObjectId('5673541534134154134513c3') +EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA = { + u'_created': datetime.datetime(2015, 12, 17, 13, 22, 56, tzinfo=tz_util.utc), + u'_etag': u'cc4643e98d3606f87bbfaaa200bfbae941b642f3', + u'_id': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID, + u'_updated': datetime.datetime(2016, 1, 7, 18, 59, 4, tzinfo=tz_util.utc), + u'category': u'assets', + u'description': u'Welcome to this curated collection of Blender Institute textures and image ' + u'resources. This collection is an on-going project, as with each project we ' + u'create a number of textures based on our own resources (photographs, scans, ' + u'etc.) or made completely from scratch. At the moment you can find all the ' + u'textures from the past Open Projects that were deemed re-usable. \r\n\r\n' + u'People who have contributed to these textures:\r\n\r\nAndrea Weikert, Andy ' + u'Goralczyk, Basse Salmela, Ben Dansie, Campbell Barton, Enrico Valenza, Ian ' + u'Hubert, Kjartan Tysdal, Manu J\xe4rvinen, Massimiliana Pulieso, Matt Ebb, ' + u'Pablo Vazquez, Rob Tuytel, Roland Hess, Sarah Feldlaufer, S\xf6nke M\xe4ter', + u'is_private': False, + u'name': u'Unittest project', + u'node_types': [ + {u'description': u'Group for texture node type', + u'dyn_schema': {u'order': {u'type': u'integer'}, + u'status': {u'allowed': [u'published', u'pending'], + u'type': u'string'}, + u'url': {u'type': u'string'}}, + u'form_schema': {}, + u'name': u'group_texture', + u'parent': [u'group_texture', u'project']}, + {u'description': u'Folder node', + u'dyn_schema': {u'notes': {u'maxlength': 256, u'type': u'string'}, + u'order': {u'type': u'integer'}, + u'status': {u'allowed': [u'published', u'pending'], + u'type': u'string'}, + u'url': {u'type': u'string'}}, + u'form_schema': {}, + u'name': u'group', + u'parent': [u'group', u'project']}, + {u'description': u'Basic Asset Type', + u'dyn_schema': { + u'attachments': {u'schema': {u'schema': {u'field': {u'type': u'string'}, + u'files': {u'schema': { + u'schema': {u'file': { + u'data_relation': { + u'embeddable': True, + u'field': u'_id', + u'resource': u'files'}, + u'type': u'objectid'}, + u'size': { + u'type': u'string'}, + u'slug': { + u'minlength': 1, + u'type': u'string'}}, + u'type': u'dict'}, + u'type': u'list'}}, + u'type': u'dict'}, + u'type': u'list'}, + u'categories': {u'type': u'string'}, + u'content_type': {u'type': u'string'}, + u'file': {u'data_relation': {u'embeddable': True, + u'field': u'_id', + u'resource': u'files'}, + u'type': u'objectid'}, + u'order': {u'type': u'integer'}, + u'status': {u'allowed': [u'published', + u'pending', + u'processing'], + u'type': u'string'}, + u'tags': {u'schema': {u'type': u'string'}, u'type': u'list'}}, + u'form_schema': {u'attachments': {u'visible': False}, + u'content_type': {u'visible': False}, + u'file': {u'visible': False}}, + u'name': u'asset', + u'parent': [u'group']}, + {u'description': u'Entrypoint to a remote or local storage solution', + u'dyn_schema': {u'backend': {u'type': u'string'}, + u'subdir': {u'type': u'string'}}, + u'form_schema': {u'backend': {}, u'subdir': {}}, + u'name': u'storage', + u'parent': [u'group', u'project'], + u'permissions': {u'groups': [{u'group': ctd.EXAMPLE_ADMIN_GROUP_ID, + u'methods': [u'GET', u'PUT', u'POST']}, + {u'group': ctd.EXAMPLE_PROJECT_READONLY_GROUP_ID, + u'methods': [u'GET']}, + {u'group': ctd.EXAMPLE_PROJECT_READONLY_GROUP2_ID, + u'methods': [u'GET']}], + u'users': [], + u'world': []}}, + {u'description': u'Comments for asset nodes, pages, etc.', + u'dyn_schema': {u'confidence': {u'type': u'float'}, + u'content': {u'minlength': 5, u'type': u'string'}, + u'is_reply': {u'type': u'boolean'}, + u'rating_negative': {u'type': u'integer'}, + u'rating_positive': {u'type': u'integer'}, + u'ratings': {u'schema': { + u'schema': {u'is_positive': {u'type': u'boolean'}, + u'user': {u'type': u'objectid'}, + u'weight': {u'type': u'integer'}}, + u'type': u'dict'}, + u'type': u'list'}, + u'status': {u'allowed': [u'published', u'flagged', u'edited'], + u'type': u'string'}}, + u'form_schema': {}, + u'name': u'comment', + u'parent': [u'asset', u'comment']}, + {u'description': u'Container for node_type post.', + u'dyn_schema': {u'categories': {u'schema': {u'type': u'string'}, + u'type': u'list'}, + u'template': {u'type': u'string'}}, + u'form_schema': {}, + u'name': u'blog', + u'parent': [u'project']}, + {u'description': u'A blog post, for any project', + u'dyn_schema': { + u'attachments': {u'schema': {u'schema': {u'field': {u'type': u'string'}, + u'files': {u'schema': { + u'schema': {u'file': { + u'data_relation': { + u'embeddable': True, + u'field': u'_id', + u'resource': u'files'}, + u'type': u'objectid'}, + u'size': { + u'type': u'string'}, + u'slug': { + u'minlength': 1, + u'type': u'string'}}, + u'type': u'dict'}, + u'type': u'list'}}, + u'type': u'dict'}, + u'type': u'list'}, + u'category': {u'type': u'string'}, + u'content': {u'maxlength': 90000, + u'minlength': 5, + u'required': True, + u'type': u'string'}, + u'status': {u'allowed': [u'published', u'pending'], + u'default': u'pending', + u'type': u'string'}, + u'url': {u'type': u'string'}}, + u'form_schema': {u'attachments': {u'visible': False}}, + u'name': u'post', + u'parent': [u'blog']}, + {u'description': u'Image Texture', + u'dyn_schema': {u'aspect_ratio': {u'type': u'float'}, + u'categories': {u'type': u'string'}, + u'files': {u'schema': {u'schema': { + u'file': {u'data_relation': {u'embeddable': True, + u'field': u'_id', + u'resource': u'files'}, + u'type': u'objectid'}, + u'is_tileable': {u'type': u'boolean'}, + u'map_type': {u'allowed': [u'color', + u'specular', + u'bump', + u'normal', + u'translucency', + u'emission', + u'alpha'], + u'type': u'string'}}, + u'type': u'dict'}, + u'type': u'list'}, + u'is_landscape': {u'type': u'boolean'}, + u'is_tileable': {u'type': u'boolean'}, + u'order': {u'type': u'integer'}, + u'resolution': {u'type': u'string'}, + u'status': {u'allowed': [u'published', + u'pending', + u'processing'], + u'type': u'string'}, + u'tags': {u'schema': {u'type': u'string'}, u'type': u'list'}}, + u'form_schema': {u'content_type': {u'visible': False}, + u'files': {u'visible': False}}, + u'name': u'texture', + u'parent': [u'group']}], + u'nodes_blog': [], + u'nodes_featured': [], + u'nodes_latest': [], + u'permissions': {u'groups': [{u'group': ctd.EXAMPLE_ADMIN_GROUP_ID, + u'methods': [u'GET', u'POST', u'PUT', u'DELETE']}], + u'users': [], + u'world': [u'GET']}, + u'status': u'published', + u'summary': u'Texture collection from all Blender Institute open projects.', + u'url': u'attachment-schema-update', + u'picture_header': ObjectId('5673f260c379cf0007b31bc4'), + u'picture_square': ObjectId('5673f256c379cf0007b31bc3'), + u'user': ctd.EXAMPLE_PROJECT_OWNER_ID} + +EXAMPLE_ASSET_NODE_OLD_ATTACHMENT_SCHEMA = { + u'_id': ObjectId('572761099837730efe8e120d'), + u'picture': ObjectId('5673f260c379cf0007b31bc4'), + u'description': u'', + u'node_type': u'asset', + u'user': ctd.EXAMPLE_PROJECT_OWNER_ID, + u'properties': { + u'status': u'published', + u'content_type': u'image', + u'file': ObjectId('5673f260c379cf0007b31bed'), + u'attachments': [{ + 'files': [ + {'slug': '01', 'file': ObjectId('5679b25ec379cf25636688f6')}, + {'slug': '02b', 'file': ObjectId('5679b308c379cf25636688f7')}, + {'slug': '03', 'file': ObjectId('5679b33bc379cf25636688f8')}, + ], + 'field': 'properties.content' + }], + }, + u'_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc), + u'name': u'Image test', + u'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID, + u'_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc), + u'_etag': u'6b8589b42c880e3626f43f3e82a5c5b946742687' +} + +EXAMPLE_PAGE_NODE_OLD_ATTACHMENT_SCHEMA = { + u'_id': ObjectId('572761099837730efe8e120a'), + u'picture': ObjectId('5673f260c379cf0007b31bc4'), + u'description': u'', + u'node_type': u'page', + u'user': ctd.EXAMPLE_PROJECT_OWNER_ID, + u'properties': { + u'status': u'published', + u'content': u'Überinteressant Verhaaltje™ voor het slapengaan.', + u'url': u'jemoeder', + u'attachments': [{ + 'files': [ + {'slug': '03', 'file': ObjectId('5679b33bc379cf256366ddd8')}, + {'slug': '04', 'file': ObjectId('5679b35bc379cf256366ddd9')}, + ], + 'field': 'properties.content' + }], + }, + u'_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc), + u'name': u'Page test', + u'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID, + u'_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc), + u'_etag': u'6b8589b42c880e3626f43f3e82a5c5b946742687' +} -class PatchCommentTest(AbstractPillarTest): +class AbstractNodeReplacementTest(AbstractPillarTest): + project_overrides = None + def setUp(self, **kwargs): AbstractPillarTest.setUp(self, **kwargs) - # Create a project that doesn't reference non-existing files, so that - # Eve can actually PUT it later without validation errors. - self.project_id, self.proj = self.ensure_project_exists(project_overrides={ - 'picture_square': None, - 'picture_header': None, + self.project_id, self.proj = self.ensure_project_exists( + project_overrides=self.project_overrides) + + self.ensure_file_exists({ + '_id': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA[u'picture_header'], + 'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID, + }) + self.ensure_file_exists({ + '_id': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA[u'picture_square'], + 'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID, }) - def test_replace_pillar_node_type_schemas(self): - from pillar.api.node_types.group import node_type_group - from pillar.cli import replace_pillar_node_type_schemas + def fetch_project_from_db(self): + with self.app.app_context(): + proj_coll = self.app.db()['projects'] + return proj_coll.find_one(self.project_id) + def add_group_permission_to_asset_node_type(self): group_perms = {u'group': ctd.EXAMPLE_PROJECT_READONLY_GROUP_ID, u'methods': [u'POST', u'PUT']} - # Assign some permissions to the node types, so we're sure they don't get overwritten. with self.app.app_context(): proj_coll = self.app.db()['projects'] @@ -31,25 +283,62 @@ class PatchCommentTest(AbstractPillarTest): {'$push': {'node_types.$.permissions.groups': group_perms}} ) + return group_perms + + +class ReplaceNodeTypesTest(AbstractNodeReplacementTest): + def test_replace_pillar_node_type_schemas(self): + from pillar.api.node_types.group import node_type_group + from pillar.cli import replace_pillar_node_type_schemas + + group_perms = self.add_group_permission_to_asset_node_type() + # Run the CLI command with self.app.test_request_context(): replace_pillar_node_type_schemas(proj_url=self.proj['url']) # Fetch the project again from MongoDB - with self.app.app_context(): - proj_coll = self.app.db()['projects'] - dbproj = proj_coll.find_one(self.project_id) - - # Perform our tests - def nt(node_type_name): - found = [nt for nt in dbproj['node_types'] - if nt['name'] == node_type_name] - return found[0] + dbproj = self.fetch_project_from_db() # Test that the node types were updated - nt_group = nt('group') + nt_group = get_node_type(dbproj, 'group') self.assertEqual(node_type_group['description'], nt_group['description']) # Test that the permissions set previously are still there. - nt_asset = nt('asset') + nt_asset = get_node_type(dbproj, 'asset') + self.assertEqual([group_perms], nt_asset['permissions']['groups']) + + +class UpgradeAttachmentSchemaTest(AbstractNodeReplacementTest): + project_overrides = EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA + + def setUp(self, **kwargs): + super(UpgradeAttachmentSchemaTest, self).setUp(**kwargs) + + self.ensure_file_exists( + {'_id': EXAMPLE_ASSET_NODE_OLD_ATTACHMENT_SCHEMA[u'properties'][u'file']}) + + for node in (EXAMPLE_ASSET_NODE_OLD_ATTACHMENT_SCHEMA, + EXAMPLE_PAGE_NODE_OLD_ATTACHMENT_SCHEMA): + for att in node[u'properties'][u'attachments']: + for filedict in att[u'files']: + self.ensure_file_exists({'_id': filedict[u'file']}) + + def test_schema_upgrade(self): + from pillar.cli import upgrade_attachment_schema + from pillar.api.node_types.asset import node_type_asset + + group_perms = self.add_group_permission_to_asset_node_type() + + with self.app.test_request_context(): + upgrade_attachment_schema(self.proj['url']) + + dbproj = self.fetch_project_from_db() + + # Test that the schemas were upgraded to the current schema. + nt_asset = get_node_type(dbproj, 'asset') + self.assertEqual(node_type_asset['dyn_schema']['attachments'], + nt_asset['dyn_schema']['attachments']) + + # Test that the permissions set previously are still there. self.assertEqual([group_perms], nt_asset['permissions']['groups'])