archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Introducing Pillar Framework

Browse Source 
Refactor of pillar-server and pillar-web into a single python package. This
simplifies the overall architecture of pillar applications.

Special thanks @sybren and @venomgfx
This commit is contained in:
Francesco Siddi
2016-08-19 09:19:06 +02:00
parent a5e92e1d87
commit 2c5dc34ea2
232 changed files with 79508 additions and 2232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

337
pillar/tests/__init__.py Normal file
View File

@@ -0,0 +1,337 @@
# -*- encoding: utf-8 -*-
import base64
import copy
import json
import logging
import datetime
import os
import sys
try:
from urllib.parse import urlencode
except ImportError:
from urllib import urlencode
from bson import ObjectId, tz_util
# Override Eve settings before importing eve.tests.
from pillar.tests import eve_test_settings
eve_test_settings.override_eve()
from eve.tests import TestMinimal
import pymongo.collection
from flask.testing import FlaskClient
import responses
from pillar.tests.common_test_data import EXAMPLE_PROJECT, EXAMPLE_FILE
import pillar
# from six:
PY3 = sys.version_info[0] == 3
if PY3:
string_type = str
text_type = str
else:
string_type = basestring
text_type = unicode
MY_PATH = os.path.dirname(os.path.abspath(__file__))
TEST_EMAIL_USER = 'koro'
TEST_EMAIL_ADDRESS = '%s@testing.blender.org' % TEST_EMAIL_USER
TEST_FULL_NAME = u'врач Сергей'
TEST_SUBCLIENT_TOKEN = 'my-subclient-token-for-pillar'
BLENDER_ID_TEST_USERID = 1896
BLENDER_ID_USER_RESPONSE = {'status': 'success',
'user': {'email': TEST_EMAIL_ADDRESS,
'full_name': TEST_FULL_NAME,
'id': BLENDER_ID_TEST_USERID},
'token_expires': 'Mon, 1 Jan 2018 01:02:03 GMT'}
class TestPillarServer(pillar.PillarServer):
def _load_flask_config(self):
super(TestPillarServer, self)._load_flask_config()
pillar_config_file = os.path.join(MY_PATH, 'config_testing.py')
self.config.from_pyfile(pillar_config_file)
def _config_logging(self):
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)-15s %(levelname)8s %(name)s %(message)s')
logging.getLogger('').setLevel(logging.DEBUG)
logging.getLogger('pillar').setLevel(logging.DEBUG)
logging.getLogger('werkzeug').setLevel(logging.DEBUG)
logging.getLogger('eve').setLevel(logging.DEBUG)
class AbstractPillarTest(TestMinimal):
pillar_server_class = TestPillarServer
def setUp(self, **kwargs):
eve_settings_file = os.path.join(MY_PATH, 'eve_test_settings.py')
kwargs['settings_file'] = eve_settings_file
os.environ['EVE_SETTINGS'] = eve_settings_file
super(AbstractPillarTest, self).setUp(**kwargs)
from eve.utils import config
config.DEBUG = True
self.app = self.pillar_server_class(os.path.dirname(os.path.dirname(__file__)))
self.app.process_extensions()
assert self.app.config['MONGO_DBNAME'] == 'pillar_test'
self.client = self.app.test_client()
assert isinstance(self.client, FlaskClient)
def tearDown(self):
super(AbstractPillarTest, self).tearDown()
# Not only delete self.app (like the superclass does),
# but also un-import the application.
del sys.modules['pillar']
remove = [modname for modname in sys.modules
if modname.startswith('pillar.')]
for modname in remove:
del sys.modules[modname]
def ensure_file_exists(self, file_overrides=None):
self.ensure_project_exists()
with self.app.test_request_context():
files_collection = self.app.data.driver.db['files']
assert isinstance(files_collection, pymongo.collection.Collection)
file = copy.deepcopy(EXAMPLE_FILE)
if file_overrides is not None:
file.update(file_overrides)
if '_id' in file and file['_id'] is None:
del file['_id']
result = files_collection.insert_one(file)
file_id = result.inserted_id
# Re-fetch from the database, so that we're sure we return the same as is stored.
# This is necessary as datetimes are rounded by MongoDB.
from_db = files_collection.find_one(file_id)
return file_id, from_db
def ensure_project_exists(self, project_overrides=None):
with self.app.test_request_context():
projects_collection = self.app.data.driver.db['projects']
assert isinstance(projects_collection, pymongo.collection.Collection)
project = copy.deepcopy(EXAMPLE_PROJECT)
if project_overrides is not None:
project.update(project_overrides)
found = projects_collection.find_one(project['_id'])
if found is None:
result = projects_collection.insert_one(project)
return result.inserted_id, project
return found['_id'], found
def create_user(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber',),
groups=None):
from pillar.api.utils.authentication import make_unique_username
with self.app.test_request_context():
users = self.app.data.driver.db['users']
assert isinstance(users, pymongo.collection.Collection)
result = users.insert_one({
'_id': ObjectId(user_id),
'_updated': datetime.datetime(2016, 4, 15, 13, 15, 11, tzinfo=tz_util.utc),
'_created': datetime.datetime(2016, 4, 15, 13, 15, 11, tzinfo=tz_util.utc),
'username': make_unique_username('tester'),
'groups': groups or [],
'roles': list(roles),
'settings': {'email_communications': 1},
'auth': [{'token': '',
'user_id': unicode(BLENDER_ID_TEST_USERID),
'provider': 'blender-id'}],
'full_name': u'คนรักของผัดไทย',
'email': TEST_EMAIL_ADDRESS
})
return result.inserted_id
def create_valid_auth_token(self, user_id, token='token'):
now = datetime.datetime.now(tz_util.utc)
future = now + datetime.timedelta(days=1)
with self.app.test_request_context():
from pillar.api.utils import authentication as auth
token_data = auth.store_token(user_id, token, future, None)
return token_data
def create_project_with_admin(self, user_id='cafef00dc379cf10c4aaceaf', roles=('subscriber', )):
"""Creates a project and a user that's member of the project's admin group.
:returns: (project_id, user_id)
:rtype: tuple
"""
project_id, proj = self.ensure_project_exists()
admin_group_id = proj['permissions']['groups'][0]['group']
user_id = self.create_user(user_id=user_id, roles=roles, groups=[admin_group_id])
return project_id, user_id
def badger(self, user_email, roles, action, srv_token=None):
"""Creates a service account, and uses it to grant or revoke a role to the user.
To skip creation of the service account, pass a srv_token.
:returns: the authentication token of the created service account.
:rtype: str
"""
if isinstance(roles, str):
roles = {roles}
# Create a service account if needed.
if srv_token is None:
from pillar.api.service import create_service_account
with self.app.test_request_context():
_, srv_token_doc = create_service_account('service@example.com',
{'badger'},
{'badger': list(roles)})
srv_token = srv_token_doc['token']
for role in roles:
self.post('/api/service/badger',
auth_token=srv_token,
json={'action': action,
'role': role,
'user_email': user_email},
expected_status=204)
return srv_token
def mock_blenderid_validate_unhappy(self):
"""Sets up Responses to mock unhappy validation flow."""
responses.add(responses.POST,
'%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'],
json={'status': 'fail'},
status=403)
def mock_blenderid_validate_happy(self):
"""Sets up Responses to mock happy validation flow."""
responses.add(responses.POST,
'%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'],
json=BLENDER_ID_USER_RESPONSE,
status=200)
def make_header(self, username, subclient_id=''):
"""Returns a Basic HTTP Authentication header value."""
return 'basic ' + base64.b64encode('%s:%s' % (username, subclient_id))
def create_standard_groups(self, additional_groups=()):
"""Creates standard admin/demo/subscriber groups, plus any additional.
:returns: mapping from group name to group ID
"""
from pillar.api import service
with self.app.test_request_context():
group_ids = {}
groups_coll = self.app.data.driver.db['groups']
for group_name in ['admin', 'demo', 'subscriber'] + list(additional_groups):
result = groups_coll.insert_one({'name': group_name})
group_ids[group_name] = result.inserted_id
service.fetch_role_to_group_id_map()
return group_ids
@staticmethod
def join_url_params(params):
"""Constructs a query string from a dictionary and appends it to a url.
Usage::
>>> AbstractPillarTest.join_url_params("pillar:5000/shots",
{"page-id": 2, "NodeType": "Shot Group"})
'pillar:5000/shots?page-id=2&NodeType=Shot+Group'
"""
if params is None:
return None
if not isinstance(params, dict):
return params
def convert_to_string(param):
if isinstance(param, dict):
return json.dumps(param, sort_keys=True)
if isinstance(param, text_type):
return param.encode('utf-8')
return param
# Pass as (key, value) pairs, so that the sorted order is maintained.
jsonified_params = [
(key, convert_to_string(params[key]))
for key in sorted(params.keys())]
return urlencode(jsonified_params)
def client_request(self, method, path, qs=None, expected_status=200, auth_token=None, json=None,
data=None, headers=None, files=None, content_type=None):
"""Performs a HTTP request to the server."""
from pillar.api.utils import dumps
import json as mod_json
headers = headers or {}
if auth_token is not None:
headers['Authorization'] = self.make_header(auth_token)
if json is not None:
data = dumps(json)
headers['Content-Type'] = 'application/json'
if files:
data = data or {}
content_type = 'multipart/form-data'
data.update(files)
resp = self.client.open(path=path, method=method, data=data, headers=headers,
content_type=content_type,
query_string=self.join_url_params(qs))
self.assertEqual(expected_status, resp.status_code,
'Expected status %i but got %i. Response: %s' % (
expected_status, resp.status_code, resp.data
))
def json():
if resp.mimetype != 'application/json':
raise TypeError('Unable to load JSON from mimetype %r' % resp.mimetype)
return mod_json.loads(resp.data)
resp.json = json
return resp
def get(self, *args, **kwargs):
return self.client_request('GET', *args, **kwargs)
def post(self, *args, **kwargs):
return self.client_request('POST', *args, **kwargs)
def put(self, *args, **kwargs):
return self.client_request('PUT', *args, **kwargs)
def delete(self, *args, **kwargs):
return self.client_request('DELETE', *args, **kwargs)
def patch(self, *args, **kwargs):
return self.client_request('PATCH', *args, **kwargs)

264
pillar/tests/common_test_data.py Normal file
View File

@@ -0,0 +1,264 @@
import datetime
from bson import tz_util, ObjectId
EXAMPLE_ADMIN_GROUP_ID = ObjectId('5596e975ea893b269af85c0e')
EXAMPLE_PROJECT_ID = ObjectId('5672beecc0261b2005ed1a33')
EXAMPLE_FILE = {u'_id': ObjectId('5672e2c1c379cf0007b31995'),
u'_updated': datetime.datetime(2016, 3, 25, 10, 28, 24, tzinfo=tz_util.utc),
u'height': 2048,
u'name': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4.png', u'format': 'png',
u'variations': [
{u'format': 'jpg', u'height': 160, u'width': 160, u'length': 8558,
u'link': 'http://localhost:8002/file-variant-h', u'content_type': 'image/jpeg',
u'md5': '--', u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-b.jpg',
u'size': 'b'},
{u'format': 'jpg', u'height': 2048, u'width': 2048, u'length': 819569,
u'link': 'http://localhost:8002/file-variant-h', u'content_type': 'image/jpeg',
u'md5': '--', u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-h.jpg',
u'size': 'h'},
{u'format': 'jpg', u'height': 64, u'width': 64, u'length': 8195,
u'link': 'http://localhost:8002/file-variant-t', u'content_type': 'image/jpeg',
u'md5': '--', u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4-t.jpg',
u'size': 't'},
],
u'filename': 'brick_dutch_soft_bump.png',
u'project': EXAMPLE_PROJECT_ID,
u'width': 2048, u'length': 6227670,
u'user': ObjectId('56264fc4fa3a250344bd10c5'),
u'content_type': 'image/png',
u'_etag': '044ce3aede2e123e261c0d8bd77212f264d4f7b0',
u'_created': datetime.datetime(2015, 12, 17, 16, 28, 49, tzinfo=tz_util.utc),
u'md5': '',
u'file_path': 'c2a5c897769ce1ef0eb10f8fa1c472bcb8e2d5a4.png',
u'backend': 'pillar',
u'link': 'http://localhost:8002/file',
u'link_expires': datetime.datetime(2016, 3, 22, 9, 28, 22, tzinfo=tz_util.utc)}
EXAMPLE_PROJECT = {
u'_created': datetime.datetime(2015, 12, 17, 13, 22, 56, tzinfo=tz_util.utc),
u'_etag': u'cc4643e98d3606f87bbfaaa200bfbae941b642f3',
u'_id': EXAMPLE_PROJECT_ID,
u'_updated': datetime.datetime(2016, 1, 7, 18, 59, 4, tzinfo=tz_util.utc),
u'category': u'assets',
u'description': u'Welcome to this curated collection of Blender Institute textures and image resources. This collection is an on-going project, as with each project we create a number of textures based on our own resources (photographs, scans, etc.) or made completely from scratch. At the moment you can find all the textures from the past Open Projects that were deemed re-usable. \r\n\r\nPeople who have contributed to these textures:\r\n\r\nAndrea Weikert, Andy Goralczyk, Basse Salmela, Ben Dansie, Campbell Barton, Enrico Valenza, Ian Hubert, Kjartan Tysdal, Manu J\xe4rvinen, Massimiliana Pulieso, Matt Ebb, Pablo Vazquez, Rob Tuytel, Roland Hess, Sarah Feldlaufer, S\xf6nke M\xe4ter',
u'is_private': False,
u'name': u'Textures',
u'node_types': [{u'description': u'Group for texture node type',
u'dyn_schema': {u'order': {u'type': u'integer'},
u'status': {u'allowed': [u'published', u'pending'],
u'type': u'string'},
u'url': {u'type': u'string'}},
u'form_schema': {u'order': {}, u'status': {}, u'url': {}},
u'name': u'group_texture',
u'parent': [u'group_texture', u'project'],
u'permissions': {}},
{u'description': u'Generic group node type edited',
u'dyn_schema': {u'notes': {u'maxlength': 256, u'type': u'string'},
u'order': {u'type': u'integer'},
u'status': {u'allowed': [u'published', u'pending'],
u'type': u'string'},
u'url': {u'type': u'string'}},
u'form_schema': {u'notes': {}, u'order': {}, u'status': {}, u'url': {}},
u'name': u'group',
u'parent': [u'group', u'project'],
u'permissions': {}},
{u'description': u'Basic Asset Type',
u'dyn_schema': {
u'attachments': {u'schema': {u'schema': {u'field': {u'type': u'string'},
u'files': {u'schema': {
u'schema': {u'file': {
u'data_relation': {
u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'size': {
u'type': u'string'},
u'slug': {
u'minlength': 1,
u'type': u'string'}},
u'type': u'dict'},
u'type': u'list'}},
u'type': u'dict'},
u'type': u'list'},
u'categories': {u'type': u'string'},
u'content_type': {u'type': u'string'},
u'file': {u'data_relation': {u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'order': {u'type': u'integer'},
u'status': {u'allowed': [u'published',
u'pending',
u'processing'],
u'type': u'string'},
u'tags': {u'schema': {u'type': u'string'}, u'type': u'list'}},
u'form_schema': {u'attachments': {u'visible': False},
u'categories': {},
u'content_type': {u'visible': False},
u'file': {u'visible': False},
u'order': {},
u'status': {},
u'tags': {}},
u'name': u'asset',
u'parent': [u'group'],
u'permissions': {}},
{u'description': u'Entrypoint to a remote or local storage solution',
u'dyn_schema': {u'backend': {u'type': u'string'},
u'subdir': {u'type': u'string'}},
u'form_schema': {u'backend': {}, u'subdir': {}},
u'name': u'storage',
u'parent': [u'group', u'project'],
u'permissions': {u'groups': [{u'group': EXAMPLE_ADMIN_GROUP_ID,
u'methods': [u'GET', u'PUT', u'POST']},
{u'group': ObjectId('5596e975ea893b269af85c0f'),
u'methods': [u'GET']},
{u'group': ObjectId('564733b56dcaf85da2faee8a'),
u'methods': [u'GET']}],
u'users': [],
u'world': []}},
{u'description': u'Comments for asset nodes, pages, etc.',
u'dyn_schema': {u'confidence': {u'type': u'float'},
u'content': {u'minlength': 5, u'type': u'string'},
u'is_reply': {u'type': u'boolean'},
u'rating_negative': {u'type': u'integer'},
u'rating_positive': {u'type': u'integer'},
u'ratings': {u'schema': {
u'schema': {u'is_positive': {u'type': u'boolean'},
u'user': {u'type': u'objectid'},
u'weight': {u'type': u'integer'}},
u'type': u'dict'},
u'type': u'list'},
u'status': {u'allowed': [u'published', u'flagged', u'edited'],
u'type': u'string'}},
u'form_schema': {u'confidence': {},
u'content': {},
u'is_reply': {},
u'rating_negative': {},
u'rating_positive': {},
u'ratings': {},
u'status': {}},
u'name': u'comment',
u'parent': [u'asset', u'comment'],
u'permissions': {}},
{u'description': u'Container for node_type post.',
u'dyn_schema': {u'categories': {u'schema': {u'type': u'string'},
u'type': u'list'},
u'template': {u'type': u'string'}},
u'form_schema': {u'categories': {}, u'template': {}},
u'name': u'blog',
u'parent': [u'project'],
u'permissions': {}},
{u'description': u'A blog post, for any project',
u'dyn_schema': {
u'attachments': {u'schema': {u'schema': {u'field': {u'type': u'string'},
u'files': {u'schema': {
u'schema': {u'file': {
u'data_relation': {
u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'size': {
u'type': u'string'},
u'slug': {
u'minlength': 1,
u'type': u'string'}},
u'type': u'dict'},
u'type': u'list'}},
u'type': u'dict'},
u'type': u'list'},
u'category': {u'type': u'string'},
u'content': {u'maxlength': 90000,
u'minlength': 5,
u'required': True,
u'type': u'string'},
u'status': {u'allowed': [u'published', u'pending'],
u'default': u'pending',
u'type': u'string'},
u'url': {u'type': u'string'}},
u'form_schema': {u'attachments': {u'visible': False},
u'category': {},
u'content': {},
u'status': {},
u'url': {}},
u'name': u'post',
u'parent': [u'blog'],
u'permissions': {}},
{u'description': u'Image Texture',
u'dyn_schema': {u'aspect_ratio': {u'type': u'float'},
u'categories': {u'type': u'string'},
u'files': {u'schema': {u'schema': {
u'file': {u'data_relation': {u'embeddable': True,
u'field': u'_id',
u'resource': u'files'},
u'type': u'objectid'},
u'is_tileable': {u'type': u'boolean'},
u'map_type': {u'allowed': [u'color',
u'specular',
u'bump',
u'normal',
u'translucency',
u'emission',
u'alpha'],
u'type': u'string'}},
u'type': u'dict'},
u'type': u'list'},
u'is_landscape': {u'type': u'boolean'},
u'is_tileable': {u'type': u'boolean'},
u'order': {u'type': u'integer'},
u'resolution': {u'type': u'string'},
u'status': {u'allowed': [u'published',
u'pending',
u'processing'],
u'type': u'string'},
u'tags': {u'schema': {u'type': u'string'}, u'type': u'list'}},
u'form_schema': {u'aspect_ratio': {},
u'categories': {},
u'content_type': {u'visible': False},
u'files': {u'visible': False},
u'is_landscape': {},
u'is_tileable': {},
u'order': {},
u'resolution': {},
u'status': {},
u'tags': {}},
u'name': u'texture',
u'parent': [u'group'],
u'permissions': {}}],
u'nodes_blog': [],
u'nodes_featured': [],
u'nodes_latest': [],
u'organization': ObjectId('55a99fb43004867fb9934f01'),
u'owners': {u'groups': [], u'users': []},
u'permissions': {u'groups': [{u'group': EXAMPLE_ADMIN_GROUP_ID,
u'methods': [u'GET', u'POST', u'PUT', u'DELETE']}],
u'users': [],
u'world': [u'GET']},
u'picture_header': ObjectId('5673f260c379cf0007b31bc4'),
u'picture_square': ObjectId('5673f256c379cf0007b31bc3'),
u'status': u'published',
u'summary': u'Texture collection from all Blender Institute open projects.',
u'url': u'textures',
u'user': ObjectId('552b066b41acdf5dec4436f2')}
EXAMPLE_NODE = {
u'_id': ObjectId('572761099837730efe8e120d'),
u'picture': ObjectId('572761f39837730efe8e1210'),
u'description': u'',
u'node_type': u'asset',
u'user': ObjectId('57164ca1983773118cbaf779'),
u'properties': {
u'status': u'published',
u'content_type': u'image',
u'file': ObjectId('572761129837730efe8e120e')
},
u'_updated': datetime.datetime(2016, 5, 2, 14, 19, 58, 0, tzinfo=tz_util.utc),
u'name': u'Image test',
u'project': EXAMPLE_PROJECT_ID,
u'_created': datetime.datetime(2016, 5, 2, 14, 19, 37, 0, tzinfo=tz_util.utc),
u'_etag': u'6b8589b42c880e3626f43f3e82a5c5b946742687'
}

11
pillar/tests/config_testing.py Normal file
View File

@@ -0,0 +1,11 @@
"""Flask configuration file for unit testing."""
BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001' # nonexistant server, no trailing slash!
DEBUG = False
TESTING = True
CDN_STORAGE_USER = 'u41508580125621'
FILESIZE_LIMIT_BYTES_NONSUBS = 20 * 2 ** 10
ROLES_FOR_UNLIMITED_UPLOADS = {u'subscriber', u'demo', u'admin'}

14
pillar/tests/eve_test_settings.py Normal file
View File

@@ -0,0 +1,14 @@
from pillar.api.eve_settings import *
MONGO_DBNAME = 'pillar_test'
def override_eve():
from eve.tests import test_settings
from eve import tests
test_settings.MONGO_HOST = MONGO_HOST
test_settings.MONGO_PORT = MONGO_PORT
test_settings.MONGO_DBNAME = MONGO_DBNAME
tests.MONGO_HOST = MONGO_HOST
tests.MONGO_DBNAME = MONGO_DBNAME