Improving initial installation process

This is still work in progress, but the new manage.py setup_db function
allows for guided creation of basic groups and for the admin user.

pillar/config.py.example

@@ -1,36 +1,39 @@
-import os
+class Development(object):
-class Config(object):
-    # Configured for GMAIL
-    MAIL_SERVER = ''
-    MAIL_PORT = 465
-    MAIL_USE_SSL = True
-    MAIL_USERNAME = ''
-    MAIL_PASSWORD = ''
-    DEFAULT_MAIL_SENDER = ''
-    RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
-
-
-class Development(Config):
-    SCHEME = 'http'
-    STORAGE_DIR = '{0}/application/static/storage'.format(
-        os.path.join(os.path.dirname(__file__)))
-    SHARED_DIR = '/storage/shared'
-    USE_X_SENDFILE = False
     PORT = 5000
     HOST = '0.0.0.0'
+    SCHEME = 'http'
     DEBUG = True
-    CDN_USE_URL_SIGNING = False
+    RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
-    CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
+    BUGSNAG_API_KEY = ''
-    CDN_SERVICE_DOMAIN = ''
-    CDN_CONTENT_SUBFOLDER = ''
-    CDN_URL_SIGNING_KEY = ''
 
+    # Settings for storage
+    STORAGE_DIR = '/data/storage/pillar'
+    SHARED_DIR = '/data/storage/shared'
+    USE_X_SENDFILE = False
+
+    # Fill in only if we are going to use a CDN-attached storage solution.
+    # Currently we use GCS and do not have enough traffic to justify that
     CDN_STORAGE_USER = ''
     CDN_STORAGE_ADDRESS = ''
     CDN_SYNC_LOGS = ''
     CDN_RSA_KEY = ''
     CDN_KNOWN_HOSTS = ''
 
+    # Credentials to access project on the Google Cloud where Google Cloud
+    # Storage is enabled (Pillar will automatically create and manage buckets)
+    GCS_CLIENT_EMAIL = ''
+    GCS_PRIVATE_KEY_P12 = ''
+    GCS_PRIVATE_KEY_PEM = ''
+    CGS_PROJECT_NAME = ''
+
+    # Fill in only if we plan to sign our urls using a the CDN
+    CDN_USE_URL_SIGNING = False
+    CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
+    CDN_SERVICE_DOMAIN = ''
+    CDN_CONTENT_SUBFOLDER = ''
+    CDN_URL_SIGNING_KEY = ''
+
+    # Settings for image processing (good defaults, should not be altered)
     UPLOADS_LOCAL_STORAGE_THUMBNAILS = {
         's': {'size': (90, 90), 'crop': True},
         'b': {'size': (160, 160), 'crop': True},
@@ -40,15 +43,24 @@ class Development(Config):
         'h': {'size': (2048, 2048), 'crop': False}
     }
 
+    # Settings for encoder (local will run FFMPEG on the server and is discouraged
+    # for production setups)
+    ENCODING_BACKEND = 'zencoder' #local, flamenco
+
+    # Zencoder is a production ready encoding solution
+    ZENCODER_API_KEY = ''
+    ZENCODER_NOTIFICATIONS_SECRET = ''
+    ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'
+
     BIN_FFPROBE ='/usr/bin/ffprobe'
     BIN_FFMPEG = '/usr/bin/ffmpeg'
     BIN_SSH = '/usr/bin/ssh'
     BIN_RSYNC = '/usr/bin/rsync'
 
+    # Settings for indexing (currently only Algolia is supported)
+    ALGOLIA_USER = ''
+    ALGOLIA_API_KEY = ''
+    ALGOLIA_INDEX_USERS = ''
 
-    GCS_CLIENT_EMAIL = ''
-    GCS_PRIVATE_KEY_P12 = ''
-    GCS_PRIVATE_KEY_PEM = ''
-    CGS_PROJECT_NAME = ''
 
 class Deployment(Development): pass

pillar/manage.py

@@ -66,6 +66,42 @@ def put_item(collection, item):
         print(item)
 
 
+@manager.command
+def setup_db():
+    """Setup the database
+    - Create admin, subscriber and demo Group collection
+    - Create admin user (must use valid blender-id credentials)
+    - Create one project
+    """
+    # groups_collection = app.data.driver.db['groups']
+    groups_list = []
+    for group in ['admin', 'subscriber', 'demo']:
+        g = {'name': group}
+        g = post_internal('groups', g)
+        groups_list.append(g[0]['_id'])
+        print("Creating group {0}".format(group))
+
+    while True:
+        admin_username = raw_input('Admin email:')
+        if len(admin_username) < 1:
+            print ("Username is too short")
+        else:
+            break
+
+    user = dict(
+        username=admin_username,
+        groups=groups_list,
+        roles=['admin', 'subscriber', 'demo'],
+        settings=dict(email_communications=1),
+        auth=[],
+        full_name=admin_username,
+        email=admin_username,
+        )
+    user = post_internal('users', user)
+    print("Created user {0}".format(user[0]['_id']))
+
+    # TODO: Create a default project
+
 @manager.command
 def clear_db():
     """Wipes the database
@@ -91,16 +127,6 @@ def upgrade_node_types():
     populate_node_types(old_ids)
 
 
-def get_id(collection, name):
-    """Returns the _id of the given collection and name"""
-    from pymongo import MongoClient
-    client = MongoClient(MONGO_HOST, 27017)
-    db = client.eve
-    node = db[collection].find({'name': name})
-    print (node[0]['_id'])
-    return node[0]['_id']
-
-
 @manager.command
 def manage_groups():
     """Take user email and group name,
@@ -264,52 +290,6 @@ def add_parent_to_nodes():
     print "Orphan {0} nodes".format(nodes_orphan)
 
 
-@manager.command
-def embed_children_in_files():
-    """Embed children file objects in to their parent"""
-    files_collection = app.data.driver.db['files']
-    for f in files_collection.find():
-        # Give some feedback
-        print "processing {0}".format(f['_id'])
-        # Proceed only if the node is a child
-        file_id = f['_id']
-        if 'parent' in f:
-            # Get the parent node
-            parent = files_collection.find_one({'_id': f['parent']})
-            if not parent:
-                print "No parent found for {0}".format(file_id)
-                files_collection.remove({'_id': file_id})
-                continue
-            parent_id = parent['_id']
-            # Prepare to loop through the properties required for a variation
-            properties = ['content_type', 'duration', 'size', 'format', 'width',
-                'height', 'length', 'md5', 'file_path']
-            variation = {}
-            # Build dict with variation properties
-            for p in properties:
-                if p in f:
-                    variation[p] = f[p]
-
-            # the variation was generated
-            if variation:
-                # If the parent file does not have a variation property
-                if 'variations' not in parent:
-                    parent['variations'] = []
-                # Append the variation to the variations
-                parent['variations'].append(variation)
-
-            # Removed internal fields that would cause validation error
-            internal_fields = ['_id', '_etag', '_updated', '_created']
-            for field in internal_fields:
-                parent.pop(field, None)
-            p = put_internal('files', parent, **{'_id': parent_id})
-            if p[0]['_status'] == 'ERR':
-                print p[0]['_issues']
-                print "PARENT: {0}".format(parent)
-                print "VARIATION: {0}".format(variation)
-                return
-
-
 @manager.command
 def remove_children_files():
     """Remove any file object with a parent field"""

pillar/settings.py

@@ -468,29 +468,6 @@ groups_schema = {
     'name': {
         'type': 'string',
         'required': True
-    },
-    'permissions': {
-        'type': 'list',
-        'required': True,
-        'schema': {
-            'type': 'dict',
-            'schema': {
-                'node_type': {
-                    'type': 'objectid',
-                    'required': True,
-                    'data_relation': {
-                        'resource': 'node_types',
-                        'field': '_id',
-                        'embeddable': True
-                    }
-                },
-                'permissions': {
-                    'type': 'list',
-                    'required': True,
-                    'allowed': ['GET', 'POST', 'UPDATE', 'DELETE']
-                }
-            }
-        }
     }
 }
 
@@ -716,5 +693,6 @@ DOMAIN = {
 
 MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost')
 MONGO_PORT = os.environ.get('MONGO_PORT', 27017)
+MONGO_DBNAME = os.environ.get('MONGO_DBNAME', 'pillar')
 CACHE_EXPIRES = 60
 HATEOAS = False