archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Introducing Pillar Framework

Browse Source 
Refactor of pillar-server and pillar-web into a single python package. This
simplifies the overall architecture of pillar applications.

Special thanks @sybren and @venomgfx
This commit is contained in:
Francesco Siddi
2016-08-19 09:19:06 +02:00
parent a5e92e1d87
commit 2c5dc34ea2
232 changed files with 79508 additions and 2232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
pillar/api/blender_cloud/__init__.py Normal file
View File

@@ -0,0 +1,30 @@
from flask import request
from werkzeug import exceptions as wz_exceptions
def blender_cloud_addon_version():
"""Returns the version of the Blender Cloud Addon, or None if not given in the request.
Uses the 'Blender-Cloud-Addon' HTTP header.
:returns: the version of the addon, as tuple (major, minor, micro)
:rtype: tuple or None
:raises: werkzeug.exceptions.BadRequest if the header is malformed.
"""
header = request.headers.get('Blender-Cloud-Addon')
if not header:
return None
parts = header.split('.')
try:
return tuple(int(part) for part in parts)
except ValueError:
raise wz_exceptions.BadRequest('Invalid Blender-Cloud-Addon header')
def setup_app(app, url_prefix):
from . import texture_libs, home_project
texture_libs.setup_app(app, url_prefix=url_prefix)
home_project.setup_app(app, url_prefix=url_prefix)

423
pillar/api/blender_cloud/home_project.py Normal file
View File

@@ -0,0 +1,423 @@
import copy
import logging
import datetime
from bson import ObjectId, tz_util
from eve.methods.get import get
from flask import Blueprint, g, current_app, request
from pillar.api import utils
from pillar.api.utils import authentication, authorization
from werkzeug import exceptions as wz_exceptions
from pillar.api.projects import utils as proj_utils
blueprint = Blueprint('blender_cloud.home_project', __name__)
log = logging.getLogger(__name__)
# Users with any of these roles will get a home project.
HOME_PROJECT_USERS = set()
# Users with any of these roles will get full write access to their home project.
HOME_PROJECT_WRITABLE_USERS = {u'subscriber', u'demo'}
HOME_PROJECT_DESCRIPTION = ('# Your home project\n\n'
'This is your home project. It allows synchronisation '
'of your Blender settings using the [Blender Cloud addon]'
'(https://cloud.blender.org/services#blender-addon).')
HOME_PROJECT_SUMMARY = 'This is your home project. Here you can sync your Blender settings!'
# HOME_PROJECT_DESCRIPTION = ('# Your home project\n\n'
# 'This is your home project. It has functionality to act '
# 'as a pastebin for text, images and other assets, and '
# 'allows synchronisation of your Blender settings.')
# HOME_PROJECT_SUMMARY = 'This is your home project. Pastebin and Blender settings sync in one!'
SYNC_GROUP_NODE_NAME = u'Blender Sync'
SYNC_GROUP_NODE_DESC = ('The [Blender Cloud Addon](https://cloud.blender.org/services'
'#blender-addon) will synchronize your Blender settings here.')
def create_blender_sync_node(project_id, admin_group_id, user_id):
"""Creates a node for Blender Sync, with explicit write access for the admin group.
Writes the node to the database.
:param project_id: ID of the home project
:type project_id: ObjectId
:param admin_group_id: ID of the admin group of the project. This group will
receive write access to the node.
:type admin_group_id: ObjectId
:param user_id: ID of the owner of the node.
:type user_id: ObjectId
:returns: The created node.
:rtype: dict
"""
log.debug('Creating sync node for project %s, user %s', project_id, user_id)
node = {
'project': ObjectId(project_id),
'node_type': 'group',
'name': SYNC_GROUP_NODE_NAME,
'user': ObjectId(user_id),
'description': SYNC_GROUP_NODE_DESC,
'properties': {'status': 'published'},
'permissions': {
'users': [],
'groups': [
{'group': ObjectId(admin_group_id),
'methods': ['GET', 'PUT', 'POST', 'DELETE']}
],
'world': [],
}
}
r, _, _, status = current_app.post_internal('nodes', node)
if status != 201:
log.warning('Unable to create Blender Sync node for home project %s: %s',
project_id, r)
raise wz_exceptions.InternalServerError('Unable to create Blender Sync node')
node.update(r)
return node
def create_home_project(user_id, write_access):
"""Creates a home project for the given user.
:param user_id: the user ID of the owner
:param write_access: whether the user has full write access to the home project.
:type write_access: bool
:returns: the project
:rtype: dict
"""
log.info('Creating home project for user %s', user_id)
overrides = {
'category': 'home',
'url': 'home',
'summary': HOME_PROJECT_SUMMARY,
'description': HOME_PROJECT_DESCRIPTION
}
# Maybe the user has a deleted home project.
proj_coll = current_app.data.driver.db['projects']
deleted_proj = proj_coll.find_one({'user': user_id, 'category': 'home', '_deleted': True})
if deleted_proj:
log.info('User %s has a deleted project %s, restoring', user_id, deleted_proj['_id'])
project = deleted_proj
else:
log.debug('User %s does not have a deleted project', user_id)
project = proj_utils.create_new_project(project_name='Home',
user_id=ObjectId(user_id),
overrides=overrides)
# Re-validate the authentication token, so that the put_internal call sees the
# new group created for the project.
authentication.validate_token()
# There are a few things in the on_insert_projects hook we need to adjust.
# Ensure that the project is private, even for admins.
project['permissions']['world'] = []
# Set up the correct node types. No need to set permissions for them,
# as the inherited project permissions are fine.
from pillar.api.node_types.group import node_type_group
from pillar.api.node_types.asset import node_type_asset
# from pillar.api.node_types.text import node_type_text
from pillar.api.node_types.comment import node_type_comment
# For non-subscribers: take away write access from the admin group,
# and grant it to certain node types.
project['permissions']['groups'][0]['methods'] = home_project_permissions(write_access)
# Everybody should be able to comment on anything in this project.
# This allows people to comment on shared images and see comments.
node_type_comment = assign_permissions(
node_type_comment,
subscriber_methods=[u'GET', u'POST'],
world_methods=[u'GET'])
project['node_types'] = [
node_type_group,
node_type_asset,
# node_type_text,
node_type_comment,
]
result, _, _, status = current_app.put_internal('projects', utils.remove_private_keys(project),
_id=project['_id'])
if status != 200:
log.error('Unable to update home project %s for user %s: %s',
project['_id'], user_id, result)
raise wz_exceptions.InternalServerError('Unable to update home project')
project.update(result)
# Create the Blender Sync node, with explicit write permissions on the node itself.
create_blender_sync_node(project['_id'],
project['permissions']['groups'][0]['group'],
user_id)
return project
def assign_permissions(node_type, subscriber_methods, world_methods):
"""Assigns permissions to the node type object.
:param node_type: a node type from pillar.api.node_types.
:type node_type: dict
:param subscriber_methods: allowed HTTP methods for users of role 'subscriber',
'demo' and 'admin'.
:type subscriber_methods: list
:param subscriber_methods: allowed HTTP methods for world
:type subscriber_methods: list
:returns: a copy of the node type, with embedded permissions
:rtype: dict
"""
from pillar.api import service
nt_with_perms = copy.deepcopy(node_type)
perms = nt_with_perms.setdefault('permissions', {})
perms['groups'] = [
{'group': service.role_to_group_id['subscriber'],
'methods': subscriber_methods[:]},
{'group': service.role_to_group_id['demo'],
'methods': subscriber_methods[:]},
{'group': service.role_to_group_id['admin'],
'methods': subscriber_methods[:]},
]
perms['world'] = world_methods[:]
return nt_with_perms
@blueprint.route('/home-project')
@authorization.require_login()
def home_project():
"""Fetches the home project, creating it if necessary.
Eve projections are supported, but at least the following fields must be present:
'permissions', 'category', 'user'
"""
user_id = g.current_user['user_id']
roles = g.current_user.get('roles', ())
log.debug('Possibly creating home project for user %s with roles %s', user_id, roles)
if HOME_PROJECT_USERS and not HOME_PROJECT_USERS.intersection(roles):
log.debug('User %s is not a subscriber, not creating home project.', user_id)
return 'No home project', 404
# Create the home project before we do the Eve query. This costs an extra round-trip
# to the database, but makes it easier to do projections correctly.
if not has_home_project(user_id):
write_access = write_access_with_roles(roles)
create_home_project(user_id, write_access)
resp, _, _, status, _ = get('projects', category=u'home', user=user_id)
if status != 200:
return utils.jsonify(resp), status
if resp['_items']:
project = resp['_items'][0]
else:
log.warning('Home project for user %s not found, while we just created it! Could be '
'due to projections and other arguments on the query string: %s',
user_id, request.query_string)
return 'No home project', 404
return utils.jsonify(project), status
def write_access_with_roles(roles):
"""Returns whether or not one of these roles grants write access to the home project.
:rtype: bool
"""
write_access = bool(not HOME_PROJECT_WRITABLE_USERS or
HOME_PROJECT_WRITABLE_USERS.intersection(roles))
return write_access
def home_project_permissions(write_access):
"""Returns the project permissions, given the write access of the user.
:rtype: list
"""
if write_access:
return [u'GET', u'PUT', u'POST', u'DELETE']
return [u'GET']
def has_home_project(user_id):
"""Returns True iff the user has a home project."""
proj_coll = current_app.data.driver.db['projects']
return proj_coll.count({'user': user_id, 'category': 'home', '_deleted': False}) > 0
def get_home_project(user_id, projection=None):
"""Returns the home project"""
proj_coll = current_app.data.driver.db['projects']
return proj_coll.find_one({'user': user_id, 'category': 'home', '_deleted': False},
projection=projection)
def is_home_project(project_id, user_id):
"""Returns True iff the given project exists and is the user's home project."""
proj_coll = current_app.data.driver.db['projects']
return proj_coll.count({'_id': project_id,
'user': user_id,
'category': 'home',
'_deleted': False}) > 0
def mark_node_updated(node_id):
"""Uses pymongo to set the node's _updated to "now"."""
now = datetime.datetime.now(tz=tz_util.utc)
nodes_coll = current_app.data.driver.db['nodes']
return nodes_coll.update_one({'_id': node_id},
{'$set': {'_updated': now}})
def get_home_project_parent_node(node, projection, name_for_log):
"""Returns a partial parent node document, but only if the node is a home project node."""
user_id = authentication.current_user_id()
if not user_id:
log.debug('%s: user not logged in.', name_for_log)
return None
parent_id = node.get('parent')
if not parent_id:
log.debug('%s: ignoring top-level node.', name_for_log)
return None
project_id = node.get('project')
if not project_id:
log.debug('%s: ignoring node without project ID', name_for_log)
return None
project_id = ObjectId(project_id)
if not is_home_project(project_id, user_id):
log.debug('%s: node not part of home project.', name_for_log)
return None
# Get the parent node for permission checking.
parent_id = ObjectId(parent_id)
nodes_coll = current_app.data.driver.db['nodes']
projection['project'] = 1
parent_node = nodes_coll.find_one(parent_id, projection=projection)
if parent_node['project'] != project_id:
log.warning('%s: User %s is trying to reference '
'parent node %s from different project %s, expected project %s.',
name_for_log, user_id, parent_id, parent_node['project'], project_id)
raise wz_exceptions.BadRequest('Trying to create cross-project links.')
return parent_node
def check_home_project_nodes_permissions(nodes):
for node in nodes:
check_home_project_node_permissions(node)
def check_home_project_node_permissions(node):
"""Grants POST access to the node when the user has POST access on its parent."""
parent_node = get_home_project_parent_node(node,
{'permissions': 1,
'project': 1,
'node_type': 1},
'check_home_project_node_permissions')
if parent_node is None or 'permissions' not in parent_node:
return
parent_id = parent_node['_id']
has_access = authorization.has_permissions('nodes', parent_node, 'POST')
if not has_access:
log.debug('check_home_project_node_permissions: No POST access to parent node %s, '
'ignoring.', parent_id)
return
# Grant access!
log.debug('check_home_project_node_permissions: POST access at parent node %s, '
'so granting POST access to new child node.', parent_id)
# Make sure the permissions of the parent node are copied to this node.
node['permissions'] = copy.deepcopy(parent_node['permissions'])
def mark_parents_as_updated(nodes):
for node in nodes:
mark_parent_as_updated(node)
def mark_parent_as_updated(node, original=None):
parent_node = get_home_project_parent_node(node,
{'permissions': 1,
'node_type': 1},
'mark_parent_as_updated')
if parent_node is None:
return
# Mark the parent node as 'updated' if this is an asset and the parent is a group.
if node.get('node_type') == 'asset' and parent_node['node_type'] == 'group':
log.debug('Node %s updated, marking parent=%s as updated too',
node['_id'], parent_node['_id'])
mark_node_updated(parent_node['_id'])
def user_changed_role(sender, user):
"""Responds to the 'user changed' signal from the Badger service.
Changes the permissions on the home project based on the 'subscriber' role.
:returns: whether this function actually made changes.
:rtype: bool
"""
user_id = user['_id']
if not has_home_project(user_id):
log.debug('User %s does not have a home project, not changing access permissions', user_id)
return
proj_coll = current_app.data.driver.db['projects']
proj = get_home_project(user_id, projection={'permissions': 1, '_id': 1})
write_access = write_access_with_roles(user['roles'])
target_permissions = home_project_permissions(write_access)
current_perms = proj['permissions']['groups'][0]['methods']
if set(current_perms) == set(target_permissions):
return False
project_id = proj['_id']
log.info('Updating permissions on user %s home project %s from %s to %s',
user_id, project_id, current_perms, target_permissions)
proj_coll.update_one({'_id': project_id},
{'$set': {'permissions.groups.0.methods': list(target_permissions)}})
return True
def setup_app(app, url_prefix):
app.register_api_blueprint(blueprint, url_prefix=url_prefix)
app.on_insert_nodes += check_home_project_nodes_permissions
app.on_inserted_nodes += mark_parents_as_updated
app.on_updated_nodes += mark_parent_as_updated
app.on_replaced_nodes += mark_parent_as_updated
from pillar.api import service
service.signal_user_changed_role.connect(user_changed_role)

146
pillar/api/blender_cloud/texture_libs.py Normal file
View File

@@ -0,0 +1,146 @@
import functools
import logging
from eve.methods.get import get
from eve.utils import config as eve_config
from flask import Blueprint, request, current_app, g
from pillar.api import utils
from pillar.api.utils.authentication import current_user_id
from pillar.api.utils.authorization import require_login
from werkzeug.datastructures import MultiDict
from werkzeug.exceptions import InternalServerError
FIRST_ADDON_VERSION_WITH_HDRI = (1, 4, 0)
TL_PROJECTION = utils.dumps({'name': 1, 'url': 1, 'permissions': 1,})
TL_SORT = utils.dumps([('name', 1)])
TEXTURE_LIBRARY_QUERY_ARGS = {
eve_config.QUERY_PROJECTION: TL_PROJECTION,
eve_config.QUERY_SORT: TL_SORT,
'max_results': 'null', # this needs to be there, or we get a KeyError.
}
blueprint = Blueprint('blender_cloud.texture_libs', __name__)
log = logging.getLogger(__name__)
def keep_fetching_texture_libraries(proj_filter):
groups = g.current_user['groups']
user_id = g.current_user['user_id']
page = 1
max_page = float('inf')
while page <= max_page:
request.args.setlist(eve_config.QUERY_PAGE, [page])
result, _, _, status, _ = get(
'projects',
{'$or': [
{'user': user_id},
{'permissions.groups.group': {'$in': groups}},
{'permissions.world': 'GET'}
]})
if status != 200:
log.warning('Error fetching texture libraries: %s', result)
raise InternalServerError('Error fetching texture libraries')
for proj in result['_items']:
if proj_filter(proj):
yield proj
# Compute the last page number we should query.
meta = result['_meta']
max_page = meta['total'] // meta['max_results']
if meta['total'] % meta['max_results'] > 0:
max_page += 1
page += 1
@blueprint.route('/texture-libraries')
@require_login()
def texture_libraries():
from . import blender_cloud_addon_version
# Use Eve method so that we get filtering on permissions for free.
# This gives all the projects that contain the required node types.
request.args = MultiDict(request.args) # allow changes; it's an ImmutableMultiDict by default.
request.args.setlist(eve_config.QUERY_PROJECTION, [TL_PROJECTION])
request.args.setlist(eve_config.QUERY_SORT, [TL_SORT])
# Determine whether to return HDRi projects too, based on the version
# of the Blender Cloud Addon. If the addon version is None, we're dealing
# with a version of the BCA that's so old it doesn't send its version along.
addon_version = blender_cloud_addon_version()
return_hdri = addon_version >= FIRST_ADDON_VERSION_WITH_HDRI
log.debug('User %s has Blender Cloud Addon version %s; return_hdri=%s',
current_user_id(), addon_version, return_hdri)
accept_as_library = functools.partial(has_texture_node, return_hdri=return_hdri)
# Construct eve-like response.
projects = list(keep_fetching_texture_libraries(accept_as_library))
result = {'_items': projects,
'_meta': {
'max_results': len(projects),
'page': 1,
'total': len(projects),
}}
return utils.jsonify(result)
def has_texture_node(proj, return_hdri=True):
"""Returns True iff the project has a top-level (group)texture node."""
nodes_collection = current_app.data.driver.db['nodes']
# See which types of nodes we support.
node_types = ['group_texture']
if return_hdri:
node_types.append('group_hdri')
count = nodes_collection.count(
{'node_type': {'$in': node_types},
'project': proj['_id'],
'parent': None})
return count > 0
def sort_by_image_width(node, original=None):
"""Sort the files in an HDRi node by image file size."""
if node.get('node_type') != 'hdri':
return
if not node.get('properties', {}).get('files'):
return
# TODO: re-enable this once all current HDRis have been saved in correct order.
# # Don't bother sorting when the files haven't changed.
# if original is not None and \
# original.get('properties', {}).get('files') == node['properties']['files']:
# return
log.info('Sorting HDRi node %s', node.get('_id', 'NO-ID'))
files_coll = current_app.data.driver.db['files']
def sort_key(file_ref):
file_doc = files_coll.find_one(file_ref['file'], projection={'length': 1})
return file_doc['length']
node['properties']['files'].sort(key=sort_key)
def sort_nodes_by_image_width(nodes):
for node in nodes:
sort_by_image_width(node)
def setup_app(app, url_prefix):
app.on_replace_nodes += sort_by_image_width
app.on_insert_nodes += sort_nodes_by_image_width
app.register_api_blueprint(blueprint, url_prefix=url_prefix)