More flexible, less error-prone configuration system.

WARNING: make a backup copy of your local config.py before pulling
this change, as Git will overwrite it without warning.

The configuration defaults to deployment settings, allowing overrides.
Overrides are read from config_local.py and from the file pointed to
by the PILLAR_CONFIG env var.

.gitignore

@@ -5,11 +5,11 @@
 *.ropeproject*
 *.swp
 
-config.py
+/pillar/config_local.py
 
 .ropeproject/*
 
-pillar/application/static/storage/
+/pillar/application/static/storage/
 /build
 /.cache
 /pillar/pillar.egg-info/

pillar/application/__init__.py

@@ -4,7 +4,8 @@ import json
 from bson import ObjectId
 from datetime import datetime
 import bugsnag
-from bugsnag.flask import handle_exceptions
+import bugsnag.flask
+import bugsnag.handlers
 from algoliasearch import algoliasearch
 from zencoder import Zencoder
 from flask import g
@@ -89,9 +90,17 @@ settings_path = os.environ.get(
     'EVE_SETTINGS', '/data/git/pillar/pillar/settings.py')
 app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth)
 
-import config
+# Load configuration from three different sources, to make it easy to override
-
+# settings with secrets, as well as for development & testing.
-app.config.from_object(config.Deployment)
+app_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+app.config.from_pyfile(os.path.join(app_root, 'config.py'), silent=False)
+app.config.from_pyfile(os.path.join(app_root, 'config_local.py'), silent=True)
+from_envvar = os.environ.get('PILLAR_CONFIG')
+if from_envvar:
+    # Don't use from_envvar, as we want different behaviour. If the envvar
+    # is not set, it's fine (i.e. silent=True), but if it is set and the
+    # configfile doesn't exist, it should error out (i.e. silent=False).
+    app.config.from_pyfile(from_envvar, silent=False)
 
 # Configure logging
 logging.basicConfig(
@@ -105,11 +114,17 @@ log.setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO)
 if app.config['DEBUG']:
     log.info('Pillar starting, debug=%s', app.config['DEBUG'])
 
-bugsnag.configure(
+# Configure Bugsnag
-    api_key=app.config['BUGSNAG_API_KEY'],
+if not app.config.get('TESTING'):
-    project_root="/data/git/pillar/pillar",
+    bugsnag.configure(
-)
+        api_key=app.config['BUGSNAG_API_KEY'],
-handle_exceptions(app)
+        project_root="/data/git/pillar/pillar",
+    )
+    bugsnag.flask.handle_exceptions(app)
+
+    bs_handler = bugsnag.handlers.BugsnagHandler()
+    bs_handler.setLevel(logging.ERROR)
+    log.addHandler(bs_handler)
 
 # Google Cloud project
 try:

pillar/config.py

@@ -0,0 +1,66 @@
+import os.path
+from collections import defaultdict
+
+RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
+
+SCHEME = 'http'
+STORAGE_DIR = '/data/storage/pillar'
+SHARED_DIR = '/data/storage/shared'
+PORT = 5000
+HOST = '0.0.0.0'
+DEBUG = False
+
+# Authentication settings
+BLENDER_ID_ENDPOINT = 'http://blender_id:8000/'
+
+CDN_USE_URL_SIGNING = True
+CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
+CDN_SERVICE_DOMAIN = 'test-blendercloud.r.worldssl.net'
+CDN_CONTENT_SUBFOLDER = ''
+CDN_URL_SIGNING_KEY = '-SECRET-'
+
+CDN_STORAGE_USER = '-SECRET'
+CDN_STORAGE_ADDRESS = 'push-11.cdnsun.com'
+CDN_SYNC_LOGS = '/data/storage/logs'
+CDN_RSA_KEY = '/data/config/cdnsun_id_rsa'
+CDN_KNOWN_HOSTS = '/data/config/known_hosts'
+
+UPLOADS_LOCAL_STORAGE_THUMBNAILS = {
+    's': {'size': (90, 90), 'crop': True},
+    'b': {'size': (160, 160), 'crop': True},
+    't': {'size': (160, 160), 'crop': False},
+    'm': {'size': (320, 320), 'crop': False},
+    'l': {'size': (1024, 1024), 'crop': False},
+    'h': {'size': (2048, 2048), 'crop': False}
+}
+
+BIN_FFPROBE = '/usr/bin/ffprobe'
+BIN_FFMPEG = '/usr/bin/ffmpeg'
+BIN_SSH = '/usr/bin/ssh'
+BIN_RSYNC = '/usr/bin/rsync'
+
+GCLOUD_APP_CREDENTIALS = os.path.join(os.path.dirname(__file__), 'google_app.json')
+GCLOUD_PROJECT = 'blender-cloud'
+
+ADMIN_USER_GROUP = '5596e975ea893b269af85c0e'
+SUBSCRIBER_USER_GROUP = '5596e975ea893b269af85c0f'
+BUGSNAG_API_KEY = ''
+
+ALGOLIA_USER = '-SECRET-'
+ALGOLIA_API_KEY = '-SECRET-'
+ALGOLIA_INDEX_USERS = 'dev_Users'
+ALGOLIA_INDEX_NODES = 'dev_Nodes'
+
+ZENCODER_API_KEY = '-SECRET-'
+ZENCODER_NOTIFICATIONS_SECRET = '-SECRET-'
+ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'
+
+ENCODING_BACKEND = 'zencoder'  # local, flamenco
+
+# Validity period of links, per file storage backend. Expressed in seconds.
+# Shouldn't be more than a year, as this isn't supported by HTTP/1.1.
+FILE_LINK_VALIDITY = defaultdict(
+    lambda: 3600 * 24 * 30,  # default of 1 month.
+    gcs=3600 * 23,  # 23 hours for Google Cloud Storage.
+)
+

pillar/config.py.example

@@ -1,79 +0,0 @@
-from collections import defaultdict
-
-
-class Development(object):
-    PORT = 5000
-    HOST = '0.0.0.0'
-    SCHEME = 'http'
-    DEBUG = True
-    RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
-    BUGSNAG_API_KEY = ''
-
-    # Authentication settings
-    BLENDER_ID_ENDPOINT = os.environ.get(
-        'BLENDER_ID_ENDPOINT', "https://www.blender.org/id").rstrip("/")
-
-    # Settings for storage
-    STORAGE_DIR = '/data/storage/pillar'
-    SHARED_DIR = '/data/storage/shared'
-    USE_X_SENDFILE = False
-
-    # Fill in only if we are going to use a CDN-attached storage solution.
-    # Currently we use GCS and do not have enough traffic to justify that
-    CDN_STORAGE_USER = ''
-    CDN_STORAGE_ADDRESS = ''
-    CDN_SYNC_LOGS = ''
-    CDN_RSA_KEY = ''
-    CDN_KNOWN_HOSTS = ''
-
-    # Credentials to access project on the Google Cloud where Google Cloud
-    # Storage is enabled (Pillar will automatically create and manage buckets)
-    GCLOUD_APP_CREDENTIALS = os.environ.get(
-        'GOOGLE_APPLICATION_CREDENTIALS', '/data/config/google_app.json')
-    GCLOUD_PROJECT = os.environ.get('GCLOUD_PROJECT', 'blender-cloud')
-
-    # Fill in only if we plan to sign our urls using a the CDN
-    CDN_USE_URL_SIGNING = False
-    CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
-    CDN_SERVICE_DOMAIN = ''
-    CDN_CONTENT_SUBFOLDER = ''
-    CDN_URL_SIGNING_KEY = ''
-
-    # Settings for image processing (good defaults, should not be altered)
-    UPLOADS_LOCAL_STORAGE_THUMBNAILS = {
-        's': {'size': (90, 90), 'crop': True},
-        'b': {'size': (160, 160), 'crop': True},
-        't': {'size': (160, 160), 'crop': False},
-        'm': {'size': (320, 320), 'crop': False},
-        'l': {'size': (1024, 1024), 'crop': False},
-        'h': {'size': (2048, 2048), 'crop': False}
-    }
-
-    # Settings for encoder (local will run FFMPEG on the server and is discouraged
-    # for production setups)
-    ENCODING_BACKEND = 'zencoder' #local, flamenco
-
-    # Zencoder is a production ready encoding solution
-    ZENCODER_API_KEY = ''
-    ZENCODER_NOTIFICATIONS_SECRET = ''
-    ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'
-
-    BIN_FFPROBE ='/usr/bin/ffprobe'
-    BIN_FFMPEG = '/usr/bin/ffmpeg'
-    BIN_SSH = '/usr/bin/ssh'
-    BIN_RSYNC = '/usr/bin/rsync'
-
-    # Settings for indexing (currently only Algolia is supported)
-    ALGOLIA_USER = ''
-    ALGOLIA_API_KEY = ''
-    ALGOLIA_INDEX_USERS = ''
-
-    # Validity period of links, per file storage backend. Expressed in seconds.
-    # Shouldn't be more than a year, as this isn't supported by HTTP/1.1.
-    FILE_LINK_VALIDITY = defaultdict(
-        lambda: 3600 * 24 * 30,  # default of 1 month.
-        gcs=3600 * 23,  # 23 hours for Google Cloud Storage.
-    )
-
-
-class Deployment(Development): pass

pillar/manage.py

@@ -32,28 +32,11 @@ MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost')
 
 @manager.command
 def runserver():
-    try:
-        import config
-        PORT = config.Development.PORT
-        HOST = config.Development.HOST
-        DEBUG = config.Development.DEBUG
-        app.config['STORAGE_DIR'] = config.Development.STORAGE_DIR
-    except ImportError:
-        # Default settings
-        PORT = 5000
-        HOST = '0.0.0.0'
-        DEBUG = True
-        app.config['STORAGE_DIR'] = '{0}/application/static/storage'.format(
-            os.path.dirname(os.path.realpath(__file__)))
-
     # Automatic creation of STORAGE_DIR path if it's missing
     if not os.path.exists(app.config['STORAGE_DIR']):
         os.makedirs(app.config['STORAGE_DIR'])
 
-    app.run(
+    app.run()
-        port=PORT,
-        host=HOST,
-        debug=DEBUG)
 
 
 def post_item(entry, data):

tests/common_test_class.py

@@ -12,7 +12,6 @@ import httpretty
 
 from common_test_data import EXAMPLE_PROJECT, EXAMPLE_FILE
 
-BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001'  # nonexistant server, no trailing slash!
 MY_PATH = os.path.dirname(os.path.abspath(__file__))
 
 TEST_EMAIL_USER = 'koro'
@@ -25,14 +24,15 @@ logging.basicConfig(
 
 class AbstractPillarTest(TestMinimal):
     def setUp(self, **kwargs):
-        settings_file = os.path.join(MY_PATH, 'common_test_settings.py')
+        eve_settings_file = os.path.join(MY_PATH, 'common_test_settings.py')
-        kwargs['settings_file'] = settings_file
+        pillar_config_file = os.path.join(MY_PATH, 'config_testing.py')
-        os.environ['EVE_SETTINGS'] = settings_file
+        kwargs['settings_file'] = eve_settings_file
+        os.environ['EVE_SETTINGS'] = eve_settings_file
+        os.environ['PILLAR_CONFIG'] = pillar_config_file
         super(AbstractPillarTest, self).setUp(**kwargs)
 
         from application import app
 
-        app.config['BLENDER_ID_ENDPOINT'] = BLENDER_ID_ENDPOINT
         logging.getLogger('application').setLevel(logging.DEBUG)
         logging.getLogger('werkzeug').setLevel(logging.DEBUG)
         logging.getLogger('eve').setLevel(logging.DEBUG)
@@ -86,7 +86,7 @@ class AbstractPillarTest(TestMinimal):
         """Sets up HTTPretty to mock unhappy validation flow."""
 
         httpretty.register_uri(httpretty.POST,
-                               '%s/u/validate_token' % BLENDER_ID_ENDPOINT,
+                               '%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'],
                                body=json.dumps(
                                    {'data': {'token': 'Token is invalid'}, 'status': 'fail'}),
                                content_type="application/json")
@@ -95,7 +95,7 @@ class AbstractPillarTest(TestMinimal):
         """Sets up HTTPretty to mock happy validation flow."""
 
         httpretty.register_uri(httpretty.POST,
-                               '%s/u/validate_token' % BLENDER_ID_ENDPOINT,
+                               '%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'],
                                body=json.dumps(
                                    {'data': {'user': {'email': TEST_EMAIL_ADDRESS, 'id': 5123}},
                                     'status': 'success'}),

tests/config_testing.py

@@ -0,0 +1,8 @@
+"""Flask configuration file for unit testing."""
+
+BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001'  # nonexistant server, no trailing slash!
+
+DEBUG = True
+TESTING = True
+
+CDN_STORAGE_USER = 'u41508580125621'