diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 14312e5c..4506c737 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -1,166 +1,20 @@ import os import json -import requests +from bson import ObjectId +from datetime import datetime import bugsnag from bugsnag.flask import handle_exceptions -from eve import Eve -from pymongo import MongoClient -from eve.auth import TokenAuth -from eve.auth import BasicAuth -from eve.io.mongo import Validator -from eve.methods.post import post_internal -from bson import ObjectId - from flask import g from flask import request from flask import url_for from flask import abort - - -from datetime import datetime -from datetime import timedelta - +from eve import Eve +from eve.io.mongo import Validator +from application.utils.authentication import validate_token +from application.utils.authentication import NewAuth RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' - -class SystemUtility(): - def __new__(cls, *args, **kwargs): - raise TypeError("Base class may not be instantiated") - - @staticmethod - def blender_id_endpoint(): - """Gets the endpoint for the authentication API. If the env variable - is defined, it's possible to override the (default) production address. - """ - return os.environ.get( - 'BLENDER_ID_ENDPOINT', "https://www.blender.org/id") - - -def validate(token): - """Validate a token against the Blender ID server. This simple lookup - returns a dictionary with the following keys: - - - message: a success message - - valid: a boolean, stating if the token is valid - - user: a dictionary with information regarding the user - """ - payload = dict( - token=token) - try: - r = requests.post("{0}/u/validate_token".format( - SystemUtility.blender_id_endpoint()), data=payload) - except requests.exceptions.ConnectionError as e: - raise e - - if r.status_code == 200: - response = r.json() - else: - response = None - return response - - -def validate_token(): - """Validate the token provided in the request and populate the current_user - flask.g object, so that permissions and access to a resource can be defined - from it. - """ - if not request.authorization: - # If no authorization headers are provided, we are getting a request - # from a non logged in user. Proceed accordingly. - return None - - current_user = {} - - token = request.authorization.username - tokens_collection = app.data.driver.db['tokens'] - - lookup = {'token': token, 'expire_time': {"$gt": datetime.now()}} - db_token = tokens_collection.find_one(lookup) - if not db_token: - # If no valid token is found, we issue a new request to the Blender ID - # to verify the validity of the token. We will get basic user info if - # the user is authorized and we will make a new token. - validation = validate(token) - if validation['status'] == 'success': - users = app.data.driver.db['users'] - email = validation['data']['user']['email'] - db_user = users.find_one({'email': email}) - # Ensure unique username - username = email.split('@')[0] - def make_unique_username(username, index=1): - """Ensure uniqueness of a username by appending an incremental - digit at the end of it. - """ - user_from_username = users.find_one({'username': username}) - if user_from_username: - if index > 1: - index += 1 - username = username[:-1] - username = "{0}{1}".format(username, index) - return make_unique_username(username, index=index) - return username - # Check for min length of username (otherwise validation fails) - username = "___{0}".format(username) if len(username) < 3 else username - username = make_unique_username(username) - - full_name = username - if not db_user: - user_data = { - 'full_name': full_name, - 'username': username, - 'email': email, - 'auth': [{ - 'provider': 'blender-id', - 'user_id': str(validation['data']['user']['id']), - 'token': ''}], - 'settings': { - 'email_communications': 1 - } - } - r = post_internal('users', user_data) - user_id = r[0]['_id'] - groups = None - else: - user_id = db_user['_id'] - groups = db_user['groups'] - - token_data = { - 'user': user_id, - 'token': token, - 'expire_time': datetime.now() + timedelta(hours=1) - } - post_internal('tokens', token_data) - current_user = dict( - user_id=user_id, - token=token, - groups=groups, - token_expire_time=datetime.now() + timedelta(hours=1)) - #return token_data - else: - return None - else: - users = app.data.driver.db['users'] - db_user = users.find_one(db_token['user']) - current_user = dict( - user_id=db_token['user'], - token=db_token['token'], - groups=db_user['groups'], - token_expire_time=db_token['expire_time']) - - setattr(g, 'current_user', current_user) - - -class NewAuth(TokenAuth): - def check_auth(self, token, allowed_roles, resource, method): - if not token: - return False - else: - validate_token() - - return True - - class ValidateCustomFields(Validator): def convert_properties(self, properties, node_schema): for prop in node_schema: @@ -219,13 +73,8 @@ class ValidateCustomFields(Validator): self._error( field, "Error validating properties") - -def post_item(entry, data): - return post_internal(entry, data) - - # We specify a settings.py file because when running on wsgi we can't detect it -# automatically. The default path (which work in Docker) can be overriden with +# automatically. The default path (which works in Docker) can be overriden with # an env variable. settings_path = os.environ.get('EVE_SETTINGS', '/data/git/pillar/pillar/settings.py') app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth) @@ -233,140 +82,17 @@ app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth) import config app.config.from_object(config.Deployment) -client = MongoClient(app.config['MONGO_HOST'], 27017) -db = client.eve bugsnag.configure( api_key = app.config['BUGSNAG_API_KEY'], project_root = "/data/git/pillar/pillar", ) handle_exceptions(app) -from utils.cdn import hash_file_path + +from application.utils.authorization import check_permissions +from application.utils.cdn import hash_file_path from application.utils.gcs import GoogleCloudStorageBucket +from application.utils.gcs import update_file_name -def update_file_name(item): - """Assign to the CGS blob the same name of the asset node. This way when - downloading an asset we get a human-readable name. - """ - - def _format_name(name, format, size=None): - # If the name already has an extention, and such extension matches the - # format, only inject the size. - root, ext = os.path.splitext(name) - size = "-{0}".format(size) if size else '' - ext = ext if len(ext) > 1 and ext[1:] == format else ".{0}".format(format) - return "{0}{1}{2}".format(root, size, ext) - - def _update_name(item, file_id): - files_collection = app.data.driver.db['files'] - f = files_collection.find_one({'_id': file_id}) - status = item['properties']['status'] - if f and f['backend'] == 'gcs' and status != 'processing': - # Process only files that are on GCS and that are not processing - try: - storage = GoogleCloudStorageBucket(str(item['project'])) - blob = storage.Get(f['file_path'], to_dict=False) - name = _format_name(item['name'], f['format']) - storage.update_name(blob, name) - try: - # Assign the same name to variations - for v in f['variations']: - blob = storage.Get(v['file_path'], to_dict=False) - name = _format_name(item['name'], v['format'], v['size']) - storage.update_name(blob, name) - except KeyError: - pass - except AttributeError: - bugsnag.notify(Exception('Missing or conflicting ids detected'), - meta_data={'nodes_info': - {'node_id': item['_id'], 'file_id': file_id}}) - - # Currently we search for 'file' and 'files' keys in the object properties. - # This could become a bit more flexible and realy on a true reference of the - # file object type from the schema. - if 'file' in item['properties']: - _update_name(item, item['properties']['file']) - - elif 'files' in item['properties']: - for f in item['properties']['files']: - _update_name(item, f['file']) - - -def check_permissions(resource, method, append_allowed_methods=False): - """Check user permissions to access a node. We look up node permissions from - world to groups to users and match them with the computed user permissions. - If there is not match, we return 403. - """ - if method != 'GET' and append_allowed_methods: - raise ValueError("append_allowed_methods only allowed with 'GET' method") - - allowed_methods = [] - - current_user = g.get('current_user', None) - - if 'permissions' in resource: - # If permissions are embedded in the node (this overrides any other - # matching permission originally set at node_type level) - resource_permissions = resource['permissions'] - else: - resource_permissions = None - if 'node_type' in resource: - if type(resource['node_type']) is dict: - # If the node_type is embedded in the document, extract permissions - # from there - computed_permissions = resource['node_type']['permissions'] - else: - # If the node_type is referenced with an ObjectID (was not embedded on - # request) query for if from the database and get the permissions - - # node_types_collection = app.data.driver.db['node_types'] - # node_type = node_types_collection.find_one(resource['node_type']) - - if type(resource['project']) is dict: - project = resource['project'] - else: - projects_collection = app.data.driver.db['projects'] - project = projects_collection.find_one(resource['project']) - node_type = next( - (item for item in project['node_types'] if item.get('name') \ - and item['name'] == resource['node_type']), None) - computed_permissions = node_type['permissions'] - else: - computed_permissions = None - - # Override computed_permissions if override is provided - if resource_permissions and computed_permissions: - for k, v in resource_permissions.iteritems(): - computed_permissions[k] = v - elif resource_permissions and not computed_permissions: - computed_permissions = resource_permissions - - if current_user: - # If the user is authenticated, proceed to compare the group permissions - for permission in computed_permissions['groups']: - if permission['group'] in current_user['groups']: - allowed_methods += permission['methods'] - if method in permission['methods'] and not append_allowed_methods: - return - - for permission in computed_permissions['users']: - if current_user['user_id'] == permission['user']: - allowed_methods += permission['methods'] - if method in permission['methods'] and not append_allowed_methods: - return - - # Check if the node is public or private. This must be set for non logged - # in users to see the content. For most BI projects this is on by default, - # while for private project this will not be set at all. - if 'world' in computed_permissions: - allowed_methods += computed_permissions['world'] - if method in computed_permissions['world'] and not append_allowed_methods: - return - - if append_allowed_methods and method in allowed_methods: - resource['allowed_methods'] = list(set(allowed_methods)) - return resource - - return None def before_returning_item_permissions(response): # Run validation process, since GET on nodes entry point is public diff --git a/pillar/application/modules/file_storage/__init__.py b/pillar/application/modules/file_storage/__init__.py index bf9e371a..6c85d2fa 100644 --- a/pillar/application/modules/file_storage/__init__.py +++ b/pillar/application/modules/file_storage/__init__.py @@ -9,7 +9,6 @@ from flask import jsonify from flask import send_from_directory from eve.methods.put import put_internal from application import app -from application import post_item from application.utils.imaging import generate_local_thumbnails from application.utils.imaging import get_video_data from application.utils.imaging import ffmpeg_encode diff --git a/pillar/application/utils/authentication.py b/pillar/application/utils/authentication.py new file mode 100644 index 00000000..a80b87c6 --- /dev/null +++ b/pillar/application/utils/authentication.py @@ -0,0 +1,146 @@ +import requests +from datetime import datetime +from datetime import timedelta +from flask import g +from flask import request +from flask import url_for +from flask import abort + +from eve.auth import TokenAuth +from eve.auth import BasicAuth + +class SystemUtility(): + def __new__(cls, *args, **kwargs): + raise TypeError("Base class may not be instantiated") + + @staticmethod + def blender_id_endpoint(): + """Gets the endpoint for the authentication API. If the env variable + is defined, it's possible to override the (default) production address. + """ + return os.environ.get( + 'BLENDER_ID_ENDPOINT', "https://www.blender.org/id") + + +def validate(token): + """Validate a token against the Blender ID server. This simple lookup + returns a dictionary with the following keys: + + - message: a success message + - valid: a boolean, stating if the token is valid + - user: a dictionary with information regarding the user + """ + payload = dict( + token=token) + try: + r = requests.post("{0}/u/validate_token".format( + SystemUtility.blender_id_endpoint()), data=payload) + except requests.exceptions.ConnectionError as e: + raise e + + if r.status_code == 200: + response = r.json() + else: + response = None + return response + + +def validate_token(): + """Validate the token provided in the request and populate the current_user + flask.g object, so that permissions and access to a resource can be defined + from it. + """ + if not request.authorization: + # If no authorization headers are provided, we are getting a request + # from a non logged in user. Proceed accordingly. + return None + + current_user = {} + + token = request.authorization.username + tokens_collection = app.data.driver.db['tokens'] + + lookup = {'token': token, 'expire_time': {"$gt": datetime.now()}} + db_token = tokens_collection.find_one(lookup) + if not db_token: + # If no valid token is found, we issue a new request to the Blender ID + # to verify the validity of the token. We will get basic user info if + # the user is authorized and we will make a new token. + validation = validate(token) + if validation['status'] == 'success': + users = app.data.driver.db['users'] + email = validation['data']['user']['email'] + db_user = users.find_one({'email': email}) + # Ensure unique username + username = email.split('@')[0] + def make_unique_username(username, index=1): + """Ensure uniqueness of a username by appending an incremental + digit at the end of it. + """ + user_from_username = users.find_one({'username': username}) + if user_from_username: + if index > 1: + index += 1 + username = username[:-1] + username = "{0}{1}".format(username, index) + return make_unique_username(username, index=index) + return username + # Check for min length of username (otherwise validation fails) + username = "___{0}".format(username) if len(username) < 3 else username + username = make_unique_username(username) + + full_name = username + if not db_user: + user_data = { + 'full_name': full_name, + 'username': username, + 'email': email, + 'auth': [{ + 'provider': 'blender-id', + 'user_id': str(validation['data']['user']['id']), + 'token': ''}], + 'settings': { + 'email_communications': 1 + } + } + r = post_internal('users', user_data) + user_id = r[0]['_id'] + groups = None + else: + user_id = db_user['_id'] + groups = db_user['groups'] + + token_data = { + 'user': user_id, + 'token': token, + 'expire_time': datetime.now() + timedelta(hours=1) + } + post_internal('tokens', token_data) + current_user = dict( + user_id=user_id, + token=token, + groups=groups, + token_expire_time=datetime.now() + timedelta(hours=1)) + #return token_data + else: + return None + else: + users = app.data.driver.db['users'] + db_user = users.find_one(db_token['user']) + current_user = dict( + user_id=db_token['user'], + token=db_token['token'], + groups=db_user['groups'], + token_expire_time=db_token['expire_time']) + + setattr(g, 'current_user', current_user) + + +class NewAuth(TokenAuth): + def check_auth(self, token, allowed_roles, resource, method): + if not token: + return False + else: + validate_token() + + return True diff --git a/pillar/application/utils/authorization.py b/pillar/application/utils/authorization.py new file mode 100644 index 00000000..d475159c --- /dev/null +++ b/pillar/application/utils/authorization.py @@ -0,0 +1,83 @@ +from flask import g +from flask import request +from flask import url_for +from flask import abort +from application import app + + +def check_permissions(resource, method, append_allowed_methods=False): + """Check user permissions to access a node. We look up node permissions from + world to groups to users and match them with the computed user permissions. + If there is not match, we return 403. + """ + if method != 'GET' and append_allowed_methods: + raise ValueError("append_allowed_methods only allowed with 'GET' method") + + allowed_methods = [] + + current_user = g.get('current_user', None) + + if 'permissions' in resource: + # If permissions are embedded in the node (this overrides any other + # matching permission originally set at node_type level) + resource_permissions = resource['permissions'] + else: + resource_permissions = None + if 'node_type' in resource: + if type(resource['node_type']) is dict: + # If the node_type is embedded in the document, extract permissions + # from there + computed_permissions = resource['node_type']['permissions'] + else: + # If the node_type is referenced with an ObjectID (was not embedded on + # request) query for if from the database and get the permissions + + # node_types_collection = app.data.driver.db['node_types'] + # node_type = node_types_collection.find_one(resource['node_type']) + + if type(resource['project']) is dict: + project = resource['project'] + else: + projects_collection = app.data.driver.db['projects'] + project = projects_collection.find_one(resource['project']) + node_type = next( + (item for item in project['node_types'] if item.get('name') \ + and item['name'] == resource['node_type']), None) + computed_permissions = node_type['permissions'] + else: + computed_permissions = None + + # Override computed_permissions if override is provided + if resource_permissions and computed_permissions: + for k, v in resource_permissions.iteritems(): + computed_permissions[k] = v + elif resource_permissions and not computed_permissions: + computed_permissions = resource_permissions + + if current_user: + # If the user is authenticated, proceed to compare the group permissions + for permission in computed_permissions['groups']: + if permission['group'] in current_user['groups']: + allowed_methods += permission['methods'] + if method in permission['methods'] and not append_allowed_methods: + return + + for permission in computed_permissions['users']: + if current_user['user_id'] == permission['user']: + allowed_methods += permission['methods'] + if method in permission['methods'] and not append_allowed_methods: + return + + # Check if the node is public or private. This must be set for non logged + # in users to see the content. For most BI projects this is on by default, + # while for private project this will not be set at all. + if 'world' in computed_permissions: + allowed_methods += computed_permissions['world'] + if method in computed_permissions['world'] and not append_allowed_methods: + return + + if append_allowed_methods and method in allowed_methods: + resource['allowed_methods'] = list(set(allowed_methods)) + return resource + + return None diff --git a/pillar/application/utils/gcs.py b/pillar/application/utils/gcs.py index f93f0f6c..067b9217 100644 --- a/pillar/application/utils/gcs.py +++ b/pillar/application/utils/gcs.py @@ -1,6 +1,7 @@ import os import time import datetime +import bugsnag from gcloud.storage.client import Client from gcloud.exceptions import NotFound from oauth2client.client import SignedJwtAssertionCredentials @@ -149,3 +150,51 @@ class GoogleCloudStorageBucket(object): """ blob.content_disposition = "attachment; filename={0}".format(name) blob.patch() + + +def update_file_name(item): + """Assign to the CGS blob the same name of the asset node. This way when + downloading an asset we get a human-readable name. + """ + + def _format_name(name, format, size=None): + # If the name already has an extention, and such extension matches the + # format, only inject the size. + root, ext = os.path.splitext(name) + size = "-{0}".format(size) if size else '' + ext = ext if len(ext) > 1 and ext[1:] == format else ".{0}".format(format) + return "{0}{1}{2}".format(root, size, ext) + + def _update_name(item, file_id): + files_collection = app.data.driver.db['files'] + f = files_collection.find_one({'_id': file_id}) + status = item['properties']['status'] + if f and f['backend'] == 'gcs' and status != 'processing': + # Process only files that are on GCS and that are not processing + try: + storage = GoogleCloudStorageBucket(str(item['project'])) + blob = storage.Get(f['file_path'], to_dict=False) + name = _format_name(item['name'], f['format']) + storage.update_name(blob, name) + try: + # Assign the same name to variations + for v in f['variations']: + blob = storage.Get(v['file_path'], to_dict=False) + name = _format_name(item['name'], v['format'], v['size']) + storage.update_name(blob, name) + except KeyError: + pass + except AttributeError: + bugsnag.notify(Exception('Missing or conflicting ids detected'), + meta_data={'nodes_info': + {'node_id': item['_id'], 'file_id': file_id}}) + + # Currently we search for 'file' and 'files' keys in the object properties. + # This could become a bit more flexible and realy on a true reference of the + # file object type from the schema. + if 'file' in item['properties']: + _update_name(item, item['properties']['file']) + + elif 'files' in item['properties']: + for f in item['properties']['files']: + _update_name(item, f['file']) diff --git a/pillar/manage.py b/pillar/manage.py index b02047b4..87f7b23d 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -5,8 +5,6 @@ from eve.methods.put import put_internal from eve.methods.post import post_internal from flask.ext.script import Manager from application import app -from application import db -from application import post_item from manage.node_types.act import node_type_act from manage.node_types.asset import node_type_asset from manage.node_types.blog import node_type_blog @@ -51,6 +49,10 @@ def runserver(): debug=DEBUG) +def post_item(entry, data): + return post_internal(entry, data) + + def put_item(collection, item): item_id = item['_id'] internal_fields = ['_id', '_etag', '_updated', '_created']