archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

New data structure for attachments.

Browse Source
This commit is contained in:
Sybren A. Stüvel 2016-10-25 15:24:06 +02:00
parent ff7101c3fe
commit 0929a80f2b
7 changed files with 436 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pillar/api/custom_field_validation.py
View File

@ -21,8 +21,13 @@ class ValidateCustomFields(Validator):
prop_type = schema_prop['type']
if prop_type == 'dict':
properties[prop] = self.convert_properties(
properties[prop], schema_prop['schema'])
try:
dict_valueschema = schema_prop['schema']
except KeyError:
# TODO: will be renamed to 'keyschema' in Cerberus 1.0
dict_valueschema = schema_prop['valueschema']
properties[prop] = self.convert_properties(properties[prop], dict_valueschema)
elif prop_type == 'list':
if properties[prop] in ['', '[]']:
properties[prop] = []

5
pillar/api/node_types/__init__.py
View File

@ -17,7 +17,10 @@ _attachments_embedded_schema = {
'valueschema': {
'type': 'dict',
'schema': {
'oid': 'objectid',
'oid': {
'type': 'objectid',
'required': True,
},
'collection': {
'type': 'string',
'allowed': ['files'],

7
pillar/api/projects/utils.py
View File

@ -90,3 +90,10 @@ def create_new_project(project_name, user_id, overrides):
log.info('Created project %s for user %s', project['_id'], user_id)
return project
def get_node_type(project, node_type_name):
"""Returns the named node type, or None if it doesn't exist."""
return next((nt for nt in project['node_types']
if nt['name'] == node_type_name), None)

93
pillar/cli.py
View File

@ -627,3 +627,96 @@ def remarkdown_comments():
log.info('identical: %i', identical)
log.info('skipped : %i', skipped)
log.info('errors : %i', errors)
@manager.command
@manager.option('-p', '--project', dest='proj_url', nargs='?',
help='Project URL')
@manager.option('-a', '--all', dest='all_projects', action='store_true', default=False,
help='Replace on all projects.')
def upgrade_attachment_schema(proj_url=None, all_projects=False):
"""Replaces the project's attachments with the new schema.
Updates both the schema definition and the nodes with attachments (asset, page, post).
"""
if bool(proj_url) == all_projects:
log.error('Use either --project or --all.')
return 1
from pillar.api.utils.authentication import force_cli_user
force_cli_user()
from pillar.api.node_types.asset import node_type_asset
from pillar.api.node_types.page import node_type_page
from pillar.api.node_types.post import node_type_post
from pillar.api.node_types import _attachments_embedded_schema
from pillar.api.utils import remove_private_keys
# Node types that support attachments
node_types = (node_type_asset, node_type_page, node_type_post)
node_type_names = {nt['name'] for nt in node_types}
db = current_app.db()
projects_coll = db['projects']
nodes_coll = db['nodes']
def handle_project(project):
log.info('Handling project %s', project['url'])
replace_schemas(project)
replace_attachments(project)
def replace_schemas(project):
for proj_nt in project['node_types']:
nt_name = proj_nt['name']
if nt_name not in node_type_names:
log.info(' - skipping node type "%s"', nt_name)
continue
log.info(' - replacing attachment schema on node type "%s"', nt_name)
proj_nt['dyn_schema']['attachments'] = copy.deepcopy(_attachments_embedded_schema)
# Use Eve to PUT, so we have schema checking.
db_proj = remove_private_keys(project)
r, _, _, status = put_internal('projects', db_proj, _id=project['_id'])
if status != 200:
log.error('Error %i storing altered project %s: %s', status, project['_id'], r)
return 4
log.info('Project saved succesfully.')
def replace_attachments(project):
log.info('Upgrading nodes for project %s', project['url'])
nodes = nodes_coll.find({
'project': project['_id'],
'node_type': {'$in': list(node_type_names)},
'properties.attachments.0': {'$exists': True},
})
for node in nodes:
log.info(' - Updating schema on node %s (%s)', node['_id'], node.get('name'))
new_atts = {}
for field_info in node[u'properties'][u'attachments']:
for attachment in field_info.get('files', []):
new_atts[attachment[u'slug']] = {u'oid': attachment[u'file']}
node[u'properties'][u'attachments'] = new_atts
# Use Eve to PUT, so we have schema checking.
db_node = remove_private_keys(node)
r, _, _, status = put_internal('nodes', db_node, _id=node['_id'])
if status != 200:
log.error('Error %i storing altered node %s: %s', status, node['_id'], r)
return
if all_projects:
for proj in projects_coll.find():
handle_project(proj)
return
proj = projects_coll.find_one({'url': proj_url})
if not proj:
log.error('Project url=%s not found', proj_url)
return 3
handle_project(proj)

18
pillar/tests/__init__.py
View File

@ -109,7 +109,11 @@ class AbstractPillarTest(TestMinimal):
del sys.modules[modname]
def ensure_file_exists(self, file_overrides=None):
self.ensure_project_exists()
if file_overrides and file_overrides.get('project'):
self.ensure_project_exists({'_id': file_overrides['project']})
else:
self.ensure_project_exists()
with self.app.test_request_context():
files_collection = self.app.data.driver.db['files']
assert isinstance(files_collection, pymongo.collection.Collection)
@ -222,7 +226,7 @@ class AbstractPillarTest(TestMinimal):
return token_data
def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )):
def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)):
"""Creates a project and a user that's member of the project's admin group.
:returns: (project_id, user_id)
@ -233,7 +237,7 @@ class AbstractPillarTest(TestMinimal):
return project_id, user_id
def create_project_admin(self, proj, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )):
def create_project_admin(self, proj, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',)):
"""Creates a user that's member of the project's admin group.
:param proj: project document, or at least a dict with permissions in it.
@ -247,6 +251,14 @@ class AbstractPillarTest(TestMinimal):
return user_id
def create_node(self, node_doc):
"""Creates a node, returning its ObjectId. """
with self.app.test_request_context():
nodes_coll = self.app.data.driver.db['nodes']
result = nodes_coll.insert_one(node_doc)
return result.inserted_id
def badger(self, user_email, roles, action, srv_token=None):
"""Creates a service account, and uses it to grant or revoke a role to the user.

5
tests/test_api/test_bcloud_home_project.py
View File

@ -404,11 +404,6 @@ class HomeProjectUserChangedRoleTest(AbstractPillarTest):
class TextureLibraryTest(AbstractHomeProjectTest):
def create_node(self, node_doc):
with self.app.test_request_context():
nodes_coll = self.app.data.driver.db['nodes']
result = nodes_coll.insert_one(node_doc)
return result.inserted_id
def setUp(self, **kwargs):
AbstractHomeProjectTest.setUp(self, **kwargs)

331
tests/test_api/test_cli.py
View File

@ -1,27 +1,279 @@
# -*- encoding: utf-8 -*-
from __future__ import absolute_import
import datetime
from bson import tz_util, ObjectId
from pillar.tests import AbstractPillarTest
from pillar.tests import common_test_data as ctd
from pillar.api.projects.utils import get_node_type
EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID = ObjectId('5673541534134154134513c3')
EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA = {
u'_created': datetime.datetime(2015, 12, 17, 13, 22, 56, tzinfo=tz_util.utc),
u'_etag': u'cc4643e98d3606f87bbfaaa200bfbae941b642f3',
u'_id': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID,
u'_updated': datetime.datetime(2016, 1, 7, 18, 59, 4, tzinfo=tz_util.utc),
u'category': u'assets',
u'description': u'Welcome to this curated collection of Blender Institute textures and image '
u'resources. This collection is an on-going project, as with each project we '
u'create a number of textures based on our own resources (photographs, scans, '
u'etc.) or made completely from scratch. At the moment you can find all the '
u'textures from the past Open Projects that were deemed re-usable. \r\n\r\n'
u'People who have contributed to these textures:\r\n\r\nAndrea Weikert, Andy '
u'Goralczyk, Basse Salmela, Ben Dansie, Campbell Barton, Enrico Valenza, Ian '
u'Hubert, Kjartan Tysdal, Manu J\xe4rvinen, Massimiliana Pulieso, Matt Ebb, '
u'Pablo Vazquez, Rob Tuytel, Roland Hess, Sarah Feldlaufer, S\xf6nke M\xe4ter',
u'is_private': False,
u'name': u'Unittest project',
u'node_types': [
{u'description': u'Group for texture node type',
u'dyn_schema': {u'order': {u'type': u'integer'},
u'status': {u'allowed': [u'published', u'pending'],
u'type': u'string'},
u'url': {u'type': u'string'}},
u'form_schema': {},
u'name': u'group_texture',
u'parent': [u'group_texture', u'project']},
{u'description': u'Folder node',
u'dyn_schema': {u'notes': {u'maxlength': 256, u'type': u'string'},
u'order': {u'type': u'integer'},
u'status': {u'allowed': [u'published', u'pending'],
u'type': u'string'},
u'url': {u'type': u'string'}},
u'form_schema': {},
u'name': u'group',
u'parent': [u'group', u'project']},
{u'description': u'Basic Asset Type',
u'dyn_schema': {
u'attachments': {u'schema': {u'schema': {u'field': {u'type': u'string'},
u'files': {u'schema': {
u'schema': {u'file': {
u'data_relation': {
u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'size': {
u'type': u'string'},
u'slug': {
u'minlength': 1,
u'type': u'string'}},
u'type': u'dict'},
u'type': u'list'}},
u'type': u'dict'},
u'type': u'list'},
u'categories': {u'type': u'string'},
u'content_type': {u'type': u'string'},
u'file': {u'data_relation': {u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'order': {u'type': u'integer'},
u'status': {u'allowed': [u'published',
u'pending',
u'processing'],
u'type': u'string'},
u'tags': {u'schema': {u'type': u'string'}, u'type': u'list'}},
u'form_schema': {u'attachments': {u'visible': False},
u'content_type': {u'visible': False},
u'file': {u'visible': False}},
u'name': u'asset',
u'parent': [u'group']},
{u'description': u'Entrypoint to a remote or local storage solution',
u'dyn_schema': {u'backend': {u'type': u'string'},
u'subdir': {u'type': u'string'}},
u'form_schema': {u'backend': {}, u'subdir': {}},
u'name': u'storage',
u'parent': [u'group', u'project'],
u'permissions': {u'groups': [{u'group': ctd.EXAMPLE_ADMIN_GROUP_ID,
u'methods': [u'GET', u'PUT', u'POST']},
{u'group': ctd.EXAMPLE_PROJECT_READONLY_GROUP_ID,
u'methods': [u'GET']},
{u'group': ctd.EXAMPLE_PROJECT_READONLY_GROUP2_ID,
u'methods': [u'GET']}],
u'users': [],
u'world': []}},
{u'description': u'Comments for asset nodes, pages, etc.',
u'dyn_schema': {u'confidence': {u'type': u'float'},
u'content': {u'minlength': 5, u'type': u'string'},
u'is_reply': {u'type': u'boolean'},
u'rating_negative': {u'type': u'integer'},
u'rating_positive': {u'type': u'integer'},
u'ratings': {u'schema': {
u'schema': {u'is_positive': {u'type': u'boolean'},
u'user': {u'type': u'objectid'},
u'weight': {u'type': u'integer'}},
u'type': u'dict'},
u'type': u'list'},
u'status': {u'allowed': [u'published', u'flagged', u'edited'],
u'type': u'string'}},
u'form_schema': {},
u'name': u'comment',
u'parent': [u'asset', u'comment']},
{u'description': u'Container for node_type post.',
u'dyn_schema': {u'categories': {u'schema': {u'type': u'string'},
u'type': u'list'},
u'template': {u'type': u'string'}},
u'form_schema': {},
u'name': u'blog',
u'parent': [u'project']},
{u'description': u'A blog post, for any project',
u'dyn_schema': {
u'attachments': {u'schema': {u'schema': {u'field': {u'type': u'string'},
u'files': {u'schema': {
u'schema': {u'file': {
u'data_relation': {
u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'size': {
u'type': u'string'},
u'slug': {
u'minlength': 1,
u'type': u'string'}},
u'type': u'dict'},
u'type': u'list'}},
u'type': u'dict'},
u'type': u'list'},
u'category': {u'type': u'string'},
u'content': {u'maxlength': 90000,
u'minlength': 5,
u'required': True,
u'type': u'string'},
u'status': {u'allowed': [u'published', u'pending'],
u'default': u'pending',
u'type': u'string'},
u'url': {u'type': u'string'}},
u'form_schema': {u'attachments': {u'visible': False}},
u'name': u'post',
u'parent': [u'blog']},
{u'description': u'Image Texture',
u'dyn_schema': {u'aspect_ratio': {u'type': u'float'},
u'categories': {u'type': u'string'},
u'files': {u'schema': {u'schema': {
u'file': {u'data_relation': {u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'is_tileable': {u'type': u'boolean'},
u'map_type': {u'allowed': [u'color',
u'specular',
u'bump',
u'normal',
u'translucency',
u'emission',
u'alpha'],
u'type': u'string'}},
u'type': u'dict'},
u'type': u'list'},
u'is_landscape': {u'type': u'boolean'},
u'is_tileable': {u'type': u'boolean'},
u'order': {u'type': u'integer'},
u'resolution': {u'type': u'string'},
u'status': {u'allowed': [u'published',
u'pending',
u'processing'],
u'type': u'string'},
u'tags': {u'schema': {u'type': u'string'}, u'type': u'list'}},
u'form_schema': {u'content_type': {u'visible': False},
u'files': {u'visible': False}},
u'name': u'texture',
u'parent': [u'group']}],
u'nodes_blog': [],
u'nodes_featured': [],
u'nodes_latest': [],
u'permissions': {u'groups': [{u'group': ctd.EXAMPLE_ADMIN_GROUP_ID,
u'methods': [u'GET', u'POST', u'PUT', u'DELETE']}],
u'users': [],
u'world': [u'GET']},
u'status': u'published',
u'summary': u'Texture collection from all Blender Institute open projects.',
u'url': u'attachment-schema-update',
u'picture_header': ObjectId('5673f260c379cf0007b31bc4'),
u'picture_square': ObjectId('5673f256c379cf0007b31bc3'),
u'user': ctd.EXAMPLE_PROJECT_OWNER_ID}
EXAMPLE_ASSET_NODE_OLD_ATTACHMENT_SCHEMA = {
u'_id': ObjectId('572761099837730efe8e120d'),
u'picture': ObjectId('5673f260c379cf0007b31bc4'),
u'description': u'',
u'node_type': u'asset',
u'user': ctd.EXAMPLE_PROJECT_OWNER_ID,
u'properties': {
u'status': u'published',
u'content_type': u'image',
u'file': ObjectId('5673f260c379cf0007b31bed'),
u'attachments': [{
'files': [
{'slug': '01', 'file': ObjectId('5679b25ec379cf25636688f6')},
{'slug': '02b', 'file': ObjectId('5679b308c379cf25636688f7')},
{'slug': '03', 'file': ObjectId('5679b33bc379cf25636688f8')},
],
'field': 'properties.content'
}],
},
u'_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc),
u'name': u'Image test',
u'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID,
u'_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc),
u'_etag': u'6b8589b42c880e3626f43f3e82a5c5b946742687'
}
EXAMPLE_PAGE_NODE_OLD_ATTACHMENT_SCHEMA = {
u'_id': ObjectId('572761099837730efe8e120a'),
u'picture': ObjectId('5673f260c379cf0007b31bc4'),
u'description': u'',
u'node_type': u'page',
u'user': ctd.EXAMPLE_PROJECT_OWNER_ID,
u'properties': {
u'status': u'published',
u'content': u'Überinteressant Verhaaltje™ voor het slapengaan.',
u'url': u'jemoeder',
u'attachments': [{
'files': [
{'slug': '03', 'file': ObjectId('5679b33bc379cf256366ddd8')},
{'slug': '04', 'file': ObjectId('5679b35bc379cf256366ddd9')},
],
'field': 'properties.content'
}],
},
u'_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc),
u'name': u'Page test',
u'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID,
u'_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc),
u'_etag': u'6b8589b42c880e3626f43f3e82a5c5b946742687'
}
class PatchCommentTest(AbstractPillarTest):
class AbstractNodeReplacementTest(AbstractPillarTest):
project_overrides = None
def setUp(self, **kwargs):
AbstractPillarTest.setUp(self, **kwargs)
# Create a project that doesn't reference non-existing files, so that
# Eve can actually PUT it later without validation errors.
self.project_id, self.proj = self.ensure_project_exists(project_overrides={
'picture_square': None,
'picture_header': None,
self.project_id, self.proj = self.ensure_project_exists(
project_overrides=self.project_overrides)
self.ensure_file_exists({
'_id': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA[u'picture_header'],
'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID,
})
self.ensure_file_exists({
'_id': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA[u'picture_square'],
'project': EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA_ID,
})
def test_replace_pillar_node_type_schemas(self):
from pillar.api.node_types.group import node_type_group
from pillar.cli import replace_pillar_node_type_schemas
def fetch_project_from_db(self):
with self.app.app_context():
proj_coll = self.app.db()['projects']
return proj_coll.find_one(self.project_id)
def add_group_permission_to_asset_node_type(self):
group_perms = {u'group': ctd.EXAMPLE_PROJECT_READONLY_GROUP_ID,
u'methods': [u'POST', u'PUT']}
# Assign some permissions to the node types, so we're sure they don't get overwritten.
with self.app.app_context():
proj_coll = self.app.db()['projects']
@ -31,25 +283,62 @@ class PatchCommentTest(AbstractPillarTest):
{'$push': {'node_types.$.permissions.groups': group_perms}}
)
return group_perms
class ReplaceNodeTypesTest(AbstractNodeReplacementTest):
def test_replace_pillar_node_type_schemas(self):
from pillar.api.node_types.group import node_type_group
from pillar.cli import replace_pillar_node_type_schemas
group_perms = self.add_group_permission_to_asset_node_type()
# Run the CLI command
with self.app.test_request_context():
replace_pillar_node_type_schemas(proj_url=self.proj['url'])
# Fetch the project again from MongoDB
with self.app.app_context():
proj_coll = self.app.db()['projects']
dbproj = proj_coll.find_one(self.project_id)
# Perform our tests
def nt(node_type_name):
found = [nt for nt in dbproj['node_types']
if nt['name'] == node_type_name]
return found[0]
dbproj = self.fetch_project_from_db()
# Test that the node types were updated
nt_group = nt('group')
nt_group = get_node_type(dbproj, 'group')
self.assertEqual(node_type_group['description'], nt_group['description'])
# Test that the permissions set previously are still there.
nt_asset = nt('asset')
nt_asset = get_node_type(dbproj, 'asset')
self.assertEqual([group_perms], nt_asset['permissions']['groups'])
class UpgradeAttachmentSchemaTest(AbstractNodeReplacementTest):
project_overrides = EXAMPLE_PROJECT_OLD_ATTACHMENT_SCHEMA
def setUp(self, **kwargs):
super(UpgradeAttachmentSchemaTest, self).setUp(**kwargs)
self.ensure_file_exists(
{'_id': EXAMPLE_ASSET_NODE_OLD_ATTACHMENT_SCHEMA[u'properties'][u'file']})
for node in (EXAMPLE_ASSET_NODE_OLD_ATTACHMENT_SCHEMA,
EXAMPLE_PAGE_NODE_OLD_ATTACHMENT_SCHEMA):
for att in node[u'properties'][u'attachments']:
for filedict in att[u'files']:
self.ensure_file_exists({'_id': filedict[u'file']})
def test_schema_upgrade(self):
from pillar.cli import upgrade_attachment_schema
from pillar.api.node_types.asset import node_type_asset
group_perms = self.add_group_permission_to_asset_node_type()
with self.app.test_request_context():
upgrade_attachment_schema(self.proj['url'])
dbproj = self.fetch_project_from_db()
# Test that the schemas were upgraded to the current schema.
nt_asset = get_node_type(dbproj, 'asset')
self.assertEqual(node_type_asset['dyn_schema']['attachments'],
nt_asset['dyn_schema']['attachments'])
# Test that the permissions set previously are still there.
self.assertEqual([group_perms], nt_asset['permissions']['groups'])