Refactored content of main __init__ into utils

pillar/application/__init__.py

@@ -1,166 +1,20 @@
 import os
 import json
-import requests
+from bson import ObjectId
+from datetime import datetime
 import bugsnag
 from bugsnag.flask import handle_exceptions
-from eve import Eve
-from pymongo import MongoClient
-from eve.auth import TokenAuth
-from eve.auth import BasicAuth
-from eve.io.mongo import Validator
-from eve.methods.post import post_internal
-from bson import ObjectId
-
 from flask import g
 from flask import request
 from flask import url_for
 from flask import abort
-
+from eve import Eve
-
+from eve.io.mongo import Validator
-from datetime import datetime
+from application.utils.authentication import validate_token
-from datetime import timedelta
+from application.utils.authentication import NewAuth
-
 
 RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
 
-
-class SystemUtility():
-    def __new__(cls, *args, **kwargs):
-        raise TypeError("Base class may not be instantiated")
-
-    @staticmethod
-    def blender_id_endpoint():
-        """Gets the endpoint for the authentication API. If the env variable
-        is defined, it's possible to override the (default) production address.
-        """
-        return os.environ.get(
-            'BLENDER_ID_ENDPOINT', "https://www.blender.org/id")
-
-
-def validate(token):
-    """Validate a token against the Blender ID server. This simple lookup
-    returns a dictionary with the following keys:
-
-    - message: a success message
-    - valid: a boolean, stating if the token is valid
-    - user: a dictionary with information regarding the user
-    """
-    payload = dict(
-        token=token)
-    try:
-        r = requests.post("{0}/u/validate_token".format(
-            SystemUtility.blender_id_endpoint()), data=payload)
-    except requests.exceptions.ConnectionError as e:
-        raise e
-
-    if r.status_code == 200:
-        response = r.json()
-    else:
-        response = None
-    return response
-
-
-def validate_token():
-    """Validate the token provided in the request and populate the current_user
-    flask.g object, so that permissions and access to a resource can be defined
-    from it.
-    """
-    if not request.authorization:
-        # If no authorization headers are provided, we are getting a request
-        # from a non logged in user. Proceed accordingly.
-        return None
-
-    current_user = {}
-
-    token = request.authorization.username
-    tokens_collection = app.data.driver.db['tokens']
-
-    lookup = {'token': token, 'expire_time': {"$gt": datetime.now()}}
-    db_token = tokens_collection.find_one(lookup)
-    if not db_token:
-        # If no valid token is found, we issue a new request to the Blender ID
-        # to verify the validity of the token. We will get basic user info if
-        # the user is authorized and we will make a new token.
-        validation = validate(token)
-        if validation['status'] == 'success':
-            users = app.data.driver.db['users']
-            email = validation['data']['user']['email']
-            db_user = users.find_one({'email': email})
-            # Ensure unique username
-            username = email.split('@')[0]
-            def make_unique_username(username, index=1):
-                """Ensure uniqueness of a username by appending an incremental
-                digit at the end of it.
-                """
-                user_from_username = users.find_one({'username': username})
-                if user_from_username:
-                    if index > 1:
-                        index += 1
-                        username = username[:-1]
-                    username = "{0}{1}".format(username, index)
-                    return make_unique_username(username, index=index)
-                return username
-            # Check for min length of username (otherwise validation fails)
-            username = "___{0}".format(username) if len(username) < 3 else username
-            username = make_unique_username(username)
-
-            full_name = username
-            if not db_user:
-                user_data = {
-                    'full_name': full_name,
-                    'username': username,
-                    'email': email,
-                    'auth': [{
-                        'provider': 'blender-id',
-                        'user_id': str(validation['data']['user']['id']),
-                        'token': ''}],
-                    'settings': {
-                        'email_communications': 1
-                    }
-                }
-                r = post_internal('users', user_data)
-                user_id = r[0]['_id']
-                groups = None
-            else:
-                user_id = db_user['_id']
-                groups = db_user['groups']
-
-            token_data = {
-                'user': user_id,
-                'token': token,
-                'expire_time': datetime.now() + timedelta(hours=1)
-            }
-            post_internal('tokens', token_data)
-            current_user = dict(
-                user_id=user_id,
-                token=token,
-                groups=groups,
-                token_expire_time=datetime.now() + timedelta(hours=1))
-            #return token_data
-        else:
-            return None
-    else:
-        users = app.data.driver.db['users']
-        db_user = users.find_one(db_token['user'])
-        current_user = dict(
-            user_id=db_token['user'],
-            token=db_token['token'],
-            groups=db_user['groups'],
-            token_expire_time=db_token['expire_time'])
-
-    setattr(g, 'current_user', current_user)
-
-
-class NewAuth(TokenAuth):
-    def check_auth(self, token, allowed_roles, resource, method):
-        if not token:
-            return False
-        else:
-            validate_token()
-
-        return True
-
-
 class ValidateCustomFields(Validator):
     def convert_properties(self, properties, node_schema):
         for prop in node_schema:
@@ -219,13 +73,8 @@ class ValidateCustomFields(Validator):
             self._error(
                 field, "Error validating properties")
 
-
-def post_item(entry, data):
-    return post_internal(entry, data)
-
-
 # We specify a settings.py file because when running on wsgi we can't detect it
-# automatically. The default path (which work in Docker) can be overriden with
+# automatically. The default path (which works in Docker) can be overriden with
 # an env variable.
 settings_path = os.environ.get('EVE_SETTINGS', '/data/git/pillar/pillar/settings.py')
 app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth)
@@ -233,140 +82,17 @@ app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth)
 import config
 app.config.from_object(config.Deployment)
 
-client = MongoClient(app.config['MONGO_HOST'], 27017)
-db = client.eve
 bugsnag.configure(
   api_key = app.config['BUGSNAG_API_KEY'],
   project_root = "/data/git/pillar/pillar",
 )
 handle_exceptions(app)
-from utils.cdn import hash_file_path
+
+from application.utils.authorization import check_permissions
+from application.utils.cdn import hash_file_path
 from application.utils.gcs import GoogleCloudStorageBucket
+from application.utils.gcs import update_file_name
 
-def update_file_name(item):
-    """Assign to the CGS blob the same name of the asset node. This way when
-    downloading an asset we get a human-readable name.
-    """
-
-    def _format_name(name, format, size=None):
-        # If the name already has an extention, and such extension matches the
-        # format, only inject the size.
-        root, ext = os.path.splitext(name)
-        size = "-{0}".format(size) if size else ''
-        ext = ext if len(ext) > 1 and ext[1:] == format else ".{0}".format(format)
-        return "{0}{1}{2}".format(root, size, ext)
-
-    def _update_name(item, file_id):
-        files_collection = app.data.driver.db['files']
-        f = files_collection.find_one({'_id': file_id})
-        status = item['properties']['status']
-        if f and f['backend'] == 'gcs' and status != 'processing':
-            # Process only files that are on GCS and that are not processing
-            try:
-                storage = GoogleCloudStorageBucket(str(item['project']))
-                blob = storage.Get(f['file_path'], to_dict=False)
-                name = _format_name(item['name'], f['format'])
-                storage.update_name(blob, name)
-                try:
-                    # Assign the same name to variations
-                    for v in f['variations']:
-                        blob = storage.Get(v['file_path'], to_dict=False)
-                        name = _format_name(item['name'], v['format'], v['size'])
-                        storage.update_name(blob, name)
-                except KeyError:
-                    pass
-            except AttributeError:
-                bugsnag.notify(Exception('Missing or conflicting ids detected'),
-                    meta_data={'nodes_info':
-                        {'node_id': item['_id'], 'file_id': file_id}})
-
-    # Currently we search for 'file' and 'files' keys in the object properties.
-    # This could become a bit more flexible and realy on a true reference of the
-    # file object type from the schema.
-    if 'file' in item['properties']:
-        _update_name(item, item['properties']['file'])
-
-    elif 'files' in item['properties']:
-        for f in item['properties']['files']:
-            _update_name(item, f['file'])
-
-
-def check_permissions(resource, method, append_allowed_methods=False):
-    """Check user permissions to access a node. We look up node permissions from
-    world to groups to users and match them with the computed user permissions.
-    If there is not match, we return 403.
-    """
-    if method != 'GET' and append_allowed_methods:
-        raise ValueError("append_allowed_methods only allowed with 'GET' method")
-
-    allowed_methods = []
-
-    current_user = g.get('current_user', None)
-
-    if 'permissions' in resource:
-        # If permissions are embedded in the node (this overrides any other
-        # matching permission originally set at node_type level)
-        resource_permissions = resource['permissions']
-    else:
-        resource_permissions = None
-    if 'node_type' in resource:
-        if type(resource['node_type']) is dict:
-            # If the node_type is embedded in the document, extract permissions
-            # from there
-            computed_permissions = resource['node_type']['permissions']
-        else:
-            # If the node_type is referenced with an ObjectID (was not embedded on
-            # request) query for if from the database and get the permissions
-
-            # node_types_collection = app.data.driver.db['node_types']
-            # node_type = node_types_collection.find_one(resource['node_type'])
-
-            if type(resource['project']) is dict:
-                project = resource['project']
-            else:
-                projects_collection = app.data.driver.db['projects']
-                project = projects_collection.find_one(resource['project'])
-            node_type = next(
-                (item for item in project['node_types'] if item.get('name') \
-                    and item['name'] == resource['node_type']), None)
-            computed_permissions = node_type['permissions']
-    else:
-        computed_permissions = None
-
-    # Override computed_permissions if override is provided
-    if resource_permissions and computed_permissions:
-        for k, v in resource_permissions.iteritems():
-            computed_permissions[k] = v
-    elif resource_permissions and not computed_permissions:
-        computed_permissions = resource_permissions
-
-    if current_user:
-        # If the user is authenticated, proceed to compare the group permissions
-        for permission in computed_permissions['groups']:
-            if permission['group'] in current_user['groups']:
-                allowed_methods += permission['methods']
-                if method in permission['methods'] and not append_allowed_methods:
-                    return
-
-        for permission in computed_permissions['users']:
-            if current_user['user_id'] == permission['user']:
-                allowed_methods += permission['methods']
-                if method in permission['methods'] and not append_allowed_methods:
-                    return
-
-    # Check if the node is public or private. This must be set for non logged
-    # in users to see the content. For most BI projects this is on by default,
-    # while for private project this will not be set at all.
-    if 'world' in computed_permissions:
-        allowed_methods += computed_permissions['world']
-        if method in computed_permissions['world'] and not append_allowed_methods:
-            return
-
-    if append_allowed_methods and method in allowed_methods:
-        resource['allowed_methods'] = list(set(allowed_methods))
-        return resource
-
-    return None
 
 def before_returning_item_permissions(response):
     # Run validation process, since GET on nodes entry point is public

pillar/application/modules/file_storage/__init__.py

@@ -9,7 +9,6 @@ from flask import jsonify
 from flask import send_from_directory
 from eve.methods.put import put_internal
 from application import app
-from application import post_item
 from application.utils.imaging import generate_local_thumbnails
 from application.utils.imaging import get_video_data
 from application.utils.imaging import ffmpeg_encode

pillar/application/utils/authentication.py

@@ -0,0 +1,146 @@
+import requests
+from datetime import datetime
+from datetime import timedelta
+from flask import g
+from flask import request
+from flask import url_for
+from flask import abort
+
+from eve.auth import TokenAuth
+from eve.auth import BasicAuth
+
+class SystemUtility():
+    def __new__(cls, *args, **kwargs):
+        raise TypeError("Base class may not be instantiated")
+
+    @staticmethod
+    def blender_id_endpoint():
+        """Gets the endpoint for the authentication API. If the env variable
+        is defined, it's possible to override the (default) production address.
+        """
+        return os.environ.get(
+            'BLENDER_ID_ENDPOINT', "https://www.blender.org/id")
+
+
+def validate(token):
+    """Validate a token against the Blender ID server. This simple lookup
+    returns a dictionary with the following keys:
+
+    - message: a success message
+    - valid: a boolean, stating if the token is valid
+    - user: a dictionary with information regarding the user
+    """
+    payload = dict(
+        token=token)
+    try:
+        r = requests.post("{0}/u/validate_token".format(
+            SystemUtility.blender_id_endpoint()), data=payload)
+    except requests.exceptions.ConnectionError as e:
+        raise e
+
+    if r.status_code == 200:
+        response = r.json()
+    else:
+        response = None
+    return response
+
+
+def validate_token():
+    """Validate the token provided in the request and populate the current_user
+    flask.g object, so that permissions and access to a resource can be defined
+    from it.
+    """
+    if not request.authorization:
+        # If no authorization headers are provided, we are getting a request
+        # from a non logged in user. Proceed accordingly.
+        return None
+
+    current_user = {}
+
+    token = request.authorization.username
+    tokens_collection = app.data.driver.db['tokens']
+
+    lookup = {'token': token, 'expire_time': {"$gt": datetime.now()}}
+    db_token = tokens_collection.find_one(lookup)
+    if not db_token:
+        # If no valid token is found, we issue a new request to the Blender ID
+        # to verify the validity of the token. We will get basic user info if
+        # the user is authorized and we will make a new token.
+        validation = validate(token)
+        if validation['status'] == 'success':
+            users = app.data.driver.db['users']
+            email = validation['data']['user']['email']
+            db_user = users.find_one({'email': email})
+            # Ensure unique username
+            username = email.split('@')[0]
+            def make_unique_username(username, index=1):
+                """Ensure uniqueness of a username by appending an incremental
+                digit at the end of it.
+                """
+                user_from_username = users.find_one({'username': username})
+                if user_from_username:
+                    if index > 1:
+                        index += 1
+                        username = username[:-1]
+                    username = "{0}{1}".format(username, index)
+                    return make_unique_username(username, index=index)
+                return username
+            # Check for min length of username (otherwise validation fails)
+            username = "___{0}".format(username) if len(username) < 3 else username
+            username = make_unique_username(username)
+
+            full_name = username
+            if not db_user:
+                user_data = {
+                    'full_name': full_name,
+                    'username': username,
+                    'email': email,
+                    'auth': [{
+                        'provider': 'blender-id',
+                        'user_id': str(validation['data']['user']['id']),
+                        'token': ''}],
+                    'settings': {
+                        'email_communications': 1
+                    }
+                }
+                r = post_internal('users', user_data)
+                user_id = r[0]['_id']
+                groups = None
+            else:
+                user_id = db_user['_id']
+                groups = db_user['groups']
+
+            token_data = {
+                'user': user_id,
+                'token': token,
+                'expire_time': datetime.now() + timedelta(hours=1)
+            }
+            post_internal('tokens', token_data)
+            current_user = dict(
+                user_id=user_id,
+                token=token,
+                groups=groups,
+                token_expire_time=datetime.now() + timedelta(hours=1))
+            #return token_data
+        else:
+            return None
+    else:
+        users = app.data.driver.db['users']
+        db_user = users.find_one(db_token['user'])
+        current_user = dict(
+            user_id=db_token['user'],
+            token=db_token['token'],
+            groups=db_user['groups'],
+            token_expire_time=db_token['expire_time'])
+
+    setattr(g, 'current_user', current_user)
+
+
+class NewAuth(TokenAuth):
+    def check_auth(self, token, allowed_roles, resource, method):
+        if not token:
+            return False
+        else:
+            validate_token()
+
+        return True

pillar/application/utils/authorization.py

@@ -0,0 +1,83 @@
+from flask import g
+from flask import request
+from flask import url_for
+from flask import abort
+from application import app
+
+
+def check_permissions(resource, method, append_allowed_methods=False):
+    """Check user permissions to access a node. We look up node permissions from
+    world to groups to users and match them with the computed user permissions.
+    If there is not match, we return 403.
+    """
+    if method != 'GET' and append_allowed_methods:
+        raise ValueError("append_allowed_methods only allowed with 'GET' method")
+
+    allowed_methods = []
+
+    current_user = g.get('current_user', None)
+
+    if 'permissions' in resource:
+        # If permissions are embedded in the node (this overrides any other
+        # matching permission originally set at node_type level)
+        resource_permissions = resource['permissions']
+    else:
+        resource_permissions = None
+    if 'node_type' in resource:
+        if type(resource['node_type']) is dict:
+            # If the node_type is embedded in the document, extract permissions
+            # from there
+            computed_permissions = resource['node_type']['permissions']
+        else:
+            # If the node_type is referenced with an ObjectID (was not embedded on
+            # request) query for if from the database and get the permissions
+
+            # node_types_collection = app.data.driver.db['node_types']
+            # node_type = node_types_collection.find_one(resource['node_type'])
+
+            if type(resource['project']) is dict:
+                project = resource['project']
+            else:
+                projects_collection = app.data.driver.db['projects']
+                project = projects_collection.find_one(resource['project'])
+            node_type = next(
+                (item for item in project['node_types'] if item.get('name') \
+                    and item['name'] == resource['node_type']), None)
+            computed_permissions = node_type['permissions']
+    else:
+        computed_permissions = None
+
+    # Override computed_permissions if override is provided
+    if resource_permissions and computed_permissions:
+        for k, v in resource_permissions.iteritems():
+            computed_permissions[k] = v
+    elif resource_permissions and not computed_permissions:
+        computed_permissions = resource_permissions
+
+    if current_user:
+        # If the user is authenticated, proceed to compare the group permissions
+        for permission in computed_permissions['groups']:
+            if permission['group'] in current_user['groups']:
+                allowed_methods += permission['methods']
+                if method in permission['methods'] and not append_allowed_methods:
+                    return
+
+        for permission in computed_permissions['users']:
+            if current_user['user_id'] == permission['user']:
+                allowed_methods += permission['methods']
+                if method in permission['methods'] and not append_allowed_methods:
+                    return
+
+    # Check if the node is public or private. This must be set for non logged
+    # in users to see the content. For most BI projects this is on by default,
+    # while for private project this will not be set at all.
+    if 'world' in computed_permissions:
+        allowed_methods += computed_permissions['world']
+        if method in computed_permissions['world'] and not append_allowed_methods:
+            return
+
+    if append_allowed_methods and method in allowed_methods:
+        resource['allowed_methods'] = list(set(allowed_methods))
+        return resource
+
+    return None

pillar/application/utils/gcs.py

@@ -1,6 +1,7 @@
 import os
 import time
 import datetime
+import bugsnag
 from gcloud.storage.client import Client
 from gcloud.exceptions import NotFound
 from oauth2client.client import SignedJwtAssertionCredentials
@@ -149,3 +150,51 @@ class GoogleCloudStorageBucket(object):
         """
         blob.content_disposition = "attachment; filename={0}".format(name)
         blob.patch()
+
+
+def update_file_name(item):
+    """Assign to the CGS blob the same name of the asset node. This way when
+    downloading an asset we get a human-readable name.
+    """
+
+    def _format_name(name, format, size=None):
+        # If the name already has an extention, and such extension matches the
+        # format, only inject the size.
+        root, ext = os.path.splitext(name)
+        size = "-{0}".format(size) if size else ''
+        ext = ext if len(ext) > 1 and ext[1:] == format else ".{0}".format(format)
+        return "{0}{1}{2}".format(root, size, ext)
+
+    def _update_name(item, file_id):
+        files_collection = app.data.driver.db['files']
+        f = files_collection.find_one({'_id': file_id})
+        status = item['properties']['status']
+        if f and f['backend'] == 'gcs' and status != 'processing':
+            # Process only files that are on GCS and that are not processing
+            try:
+                storage = GoogleCloudStorageBucket(str(item['project']))
+                blob = storage.Get(f['file_path'], to_dict=False)
+                name = _format_name(item['name'], f['format'])
+                storage.update_name(blob, name)
+                try:
+                    # Assign the same name to variations
+                    for v in f['variations']:
+                        blob = storage.Get(v['file_path'], to_dict=False)
+                        name = _format_name(item['name'], v['format'], v['size'])
+                        storage.update_name(blob, name)
+                except KeyError:
+                    pass
+            except AttributeError:
+                bugsnag.notify(Exception('Missing or conflicting ids detected'),
+                    meta_data={'nodes_info':
+                        {'node_id': item['_id'], 'file_id': file_id}})
+
+    # Currently we search for 'file' and 'files' keys in the object properties.
+    # This could become a bit more flexible and realy on a true reference of the
+    # file object type from the schema.
+    if 'file' in item['properties']:
+        _update_name(item, item['properties']['file'])
+
+    elif 'files' in item['properties']:
+        for f in item['properties']['files']:
+            _update_name(item, f['file'])

pillar/manage.py

@@ -5,8 +5,6 @@ from eve.methods.put import put_internal
 from eve.methods.post import post_internal
 from flask.ext.script import Manager
 from application import app
-from application import db
-from application import post_item
 from manage.node_types.act import node_type_act
 from manage.node_types.asset import node_type_asset
 from manage.node_types.blog import node_type_blog
@@ -51,6 +49,10 @@ def runserver():
         debug=DEBUG)
 
 
+def post_item(entry, data):
+    return post_internal(entry, data)
+
+
 def put_item(collection, item):
     item_id = item['_id']
     internal_fields = ['_id', '_etag', '_updated', '_created']