From d76c47afd550ac1f397effbb405a9b310caa84ba Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Thu, 25 Feb 2016 12:43:46 +0100 Subject: [PATCH] Improving initial installation process This is still work in progress, but the new manage.py setup_db function allows for guided creation of basic groups and for the admin user. --- pillar/config.py.example | 66 ++++++++++++++++------------ pillar/manage.py | 92 ++++++++++++++++------------------------ pillar/settings.py | 24 +---------- 3 files changed, 76 insertions(+), 106 deletions(-) diff --git a/pillar/config.py.example b/pillar/config.py.example index 7e0dbad3..0670fff2 100644 --- a/pillar/config.py.example +++ b/pillar/config.py.example @@ -1,36 +1,39 @@ -import os -class Config(object): - # Configured for GMAIL - MAIL_SERVER = '' - MAIL_PORT = 465 - MAIL_USE_SSL = True - MAIL_USERNAME = '' - MAIL_PASSWORD = '' - DEFAULT_MAIL_SENDER = '' - RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' - - -class Development(Config): - SCHEME = 'http' - STORAGE_DIR = '{0}/application/static/storage'.format( - os.path.join(os.path.dirname(__file__))) - SHARED_DIR = '/storage/shared' - USE_X_SENDFILE = False +class Development(object): PORT = 5000 HOST = '0.0.0.0' + SCHEME = 'http' DEBUG = True - CDN_USE_URL_SIGNING = False - CDN_SERVICE_DOMAIN_PROTOCOL = 'https' - CDN_SERVICE_DOMAIN = '' - CDN_CONTENT_SUBFOLDER = '' - CDN_URL_SIGNING_KEY = '' + RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' + BUGSNAG_API_KEY = '' + # Settings for storage + STORAGE_DIR = '/data/storage/pillar' + SHARED_DIR = '/data/storage/shared' + USE_X_SENDFILE = False + + # Fill in only if we are going to use a CDN-attached storage solution. + # Currently we use GCS and do not have enough traffic to justify that CDN_STORAGE_USER = '' CDN_STORAGE_ADDRESS = '' CDN_SYNC_LOGS = '' CDN_RSA_KEY = '' CDN_KNOWN_HOSTS = '' + # Credentials to access project on the Google Cloud where Google Cloud + # Storage is enabled (Pillar will automatically create and manage buckets) + GCS_CLIENT_EMAIL = '' + GCS_PRIVATE_KEY_P12 = '' + GCS_PRIVATE_KEY_PEM = '' + CGS_PROJECT_NAME = '' + + # Fill in only if we plan to sign our urls using a the CDN + CDN_USE_URL_SIGNING = False + CDN_SERVICE_DOMAIN_PROTOCOL = 'https' + CDN_SERVICE_DOMAIN = '' + CDN_CONTENT_SUBFOLDER = '' + CDN_URL_SIGNING_KEY = '' + + # Settings for image processing (good defaults, should not be altered) UPLOADS_LOCAL_STORAGE_THUMBNAILS = { 's': {'size': (90, 90), 'crop': True}, 'b': {'size': (160, 160), 'crop': True}, @@ -40,15 +43,24 @@ class Development(Config): 'h': {'size': (2048, 2048), 'crop': False} } + # Settings for encoder (local will run FFMPEG on the server and is discouraged + # for production setups) + ENCODING_BACKEND = 'zencoder' #local, flamenco + + # Zencoder is a production ready encoding solution + ZENCODER_API_KEY = '' + ZENCODER_NOTIFICATIONS_SECRET = '' + ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/' + BIN_FFPROBE ='/usr/bin/ffprobe' BIN_FFMPEG = '/usr/bin/ffmpeg' BIN_SSH = '/usr/bin/ssh' BIN_RSYNC = '/usr/bin/rsync' + # Settings for indexing (currently only Algolia is supported) + ALGOLIA_USER = '' + ALGOLIA_API_KEY = '' + ALGOLIA_INDEX_USERS = '' - GCS_CLIENT_EMAIL = '' - GCS_PRIVATE_KEY_P12 = '' - GCS_PRIVATE_KEY_PEM = '' - CGS_PROJECT_NAME = '' class Deployment(Development): pass diff --git a/pillar/manage.py b/pillar/manage.py index fcb35662..5e21728c 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -66,6 +66,42 @@ def put_item(collection, item): print(item) +@manager.command +def setup_db(): + """Setup the database + - Create admin, subscriber and demo Group collection + - Create admin user (must use valid blender-id credentials) + - Create one project + """ + # groups_collection = app.data.driver.db['groups'] + groups_list = [] + for group in ['admin', 'subscriber', 'demo']: + g = {'name': group} + g = post_internal('groups', g) + groups_list.append(g[0]['_id']) + print("Creating group {0}".format(group)) + + while True: + admin_username = raw_input('Admin email:') + if len(admin_username) < 1: + print ("Username is too short") + else: + break + + user = dict( + username=admin_username, + groups=groups_list, + roles=['admin', 'subscriber', 'demo'], + settings=dict(email_communications=1), + auth=[], + full_name=admin_username, + email=admin_username, + ) + user = post_internal('users', user) + print("Created user {0}".format(user[0]['_id'])) + + # TODO: Create a default project + @manager.command def clear_db(): """Wipes the database @@ -91,16 +127,6 @@ def upgrade_node_types(): populate_node_types(old_ids) -def get_id(collection, name): - """Returns the _id of the given collection and name""" - from pymongo import MongoClient - client = MongoClient(MONGO_HOST, 27017) - db = client.eve - node = db[collection].find({'name': name}) - print (node[0]['_id']) - return node[0]['_id'] - - @manager.command def manage_groups(): """Take user email and group name, @@ -264,52 +290,6 @@ def add_parent_to_nodes(): print "Orphan {0} nodes".format(nodes_orphan) -@manager.command -def embed_children_in_files(): - """Embed children file objects in to their parent""" - files_collection = app.data.driver.db['files'] - for f in files_collection.find(): - # Give some feedback - print "processing {0}".format(f['_id']) - # Proceed only if the node is a child - file_id = f['_id'] - if 'parent' in f: - # Get the parent node - parent = files_collection.find_one({'_id': f['parent']}) - if not parent: - print "No parent found for {0}".format(file_id) - files_collection.remove({'_id': file_id}) - continue - parent_id = parent['_id'] - # Prepare to loop through the properties required for a variation - properties = ['content_type', 'duration', 'size', 'format', 'width', - 'height', 'length', 'md5', 'file_path'] - variation = {} - # Build dict with variation properties - for p in properties: - if p in f: - variation[p] = f[p] - - # the variation was generated - if variation: - # If the parent file does not have a variation property - if 'variations' not in parent: - parent['variations'] = [] - # Append the variation to the variations - parent['variations'].append(variation) - - # Removed internal fields that would cause validation error - internal_fields = ['_id', '_etag', '_updated', '_created'] - for field in internal_fields: - parent.pop(field, None) - p = put_internal('files', parent, **{'_id': parent_id}) - if p[0]['_status'] == 'ERR': - print p[0]['_issues'] - print "PARENT: {0}".format(parent) - print "VARIATION: {0}".format(variation) - return - - @manager.command def remove_children_files(): """Remove any file object with a parent field""" diff --git a/pillar/settings.py b/pillar/settings.py index 48322733..331a0309 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -468,29 +468,6 @@ groups_schema = { 'name': { 'type': 'string', 'required': True - }, - 'permissions': { - 'type': 'list', - 'required': True, - 'schema': { - 'type': 'dict', - 'schema': { - 'node_type': { - 'type': 'objectid', - 'required': True, - 'data_relation': { - 'resource': 'node_types', - 'field': '_id', - 'embeddable': True - } - }, - 'permissions': { - 'type': 'list', - 'required': True, - 'allowed': ['GET', 'POST', 'UPDATE', 'DELETE'] - } - } - } } } @@ -716,5 +693,6 @@ DOMAIN = { MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost') MONGO_PORT = os.environ.get('MONGO_PORT', 27017) +MONGO_DBNAME = os.environ.get('MONGO_DBNAME', 'pillar') CACHE_EXPIRES = 60 HATEOAS = False