From 465b14560995b27fef423a7b0ff269611e9b49a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 4 Apr 2016 14:59:11 +0200 Subject: [PATCH] More flexible, less error-prone configuration system. WARNING: make a backup copy of your local config.py before pulling this change, as Git will overwrite it without warning. The configuration defaults to deployment settings, allowing overrides. Overrides are read from config_local.py and from the file pointed to by the PILLAR_CONFIG env var. --- .gitignore | 4 +- pillar/application/__init__.py | 33 ++++++++++---- pillar/config.py | 66 ++++++++++++++++++++++++++++ pillar/config.py.example | 79 ---------------------------------- pillar/manage.py | 19 +------- tests/common_test_class.py | 14 +++--- tests/config_testing.py | 8 ++++ 7 files changed, 108 insertions(+), 115 deletions(-) create mode 100644 pillar/config.py delete mode 100644 pillar/config.py.example create mode 100644 tests/config_testing.py diff --git a/.gitignore b/.gitignore index 85130cb4..5404cbac 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,11 @@ *.ropeproject* *.swp -config.py +/pillar/config_local.py .ropeproject/* -pillar/application/static/storage/ +/pillar/application/static/storage/ /build /.cache /pillar/pillar.egg-info/ diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index c6778a71..3de98b41 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -4,7 +4,8 @@ import json from bson import ObjectId from datetime import datetime import bugsnag -from bugsnag.flask import handle_exceptions +import bugsnag.flask +import bugsnag.handlers from algoliasearch import algoliasearch from zencoder import Zencoder from flask import g @@ -89,9 +90,17 @@ settings_path = os.environ.get( 'EVE_SETTINGS', '/data/git/pillar/pillar/settings.py') app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth) -import config - -app.config.from_object(config.Deployment) +# Load configuration from three different sources, to make it easy to override +# settings with secrets, as well as for development & testing. +app_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +app.config.from_pyfile(os.path.join(app_root, 'config.py'), silent=False) +app.config.from_pyfile(os.path.join(app_root, 'config_local.py'), silent=True) +from_envvar = os.environ.get('PILLAR_CONFIG') +if from_envvar: + # Don't use from_envvar, as we want different behaviour. If the envvar + # is not set, it's fine (i.e. silent=True), but if it is set and the + # configfile doesn't exist, it should error out (i.e. silent=False). + app.config.from_pyfile(from_envvar, silent=False) # Configure logging logging.basicConfig( @@ -105,11 +114,17 @@ log.setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO) if app.config['DEBUG']: log.info('Pillar starting, debug=%s', app.config['DEBUG']) -bugsnag.configure( - api_key=app.config['BUGSNAG_API_KEY'], - project_root="/data/git/pillar/pillar", -) -handle_exceptions(app) +# Configure Bugsnag +if not app.config.get('TESTING'): + bugsnag.configure( + api_key=app.config['BUGSNAG_API_KEY'], + project_root="/data/git/pillar/pillar", + ) + bugsnag.flask.handle_exceptions(app) + + bs_handler = bugsnag.handlers.BugsnagHandler() + bs_handler.setLevel(logging.ERROR) + log.addHandler(bs_handler) # Google Cloud project try: diff --git a/pillar/config.py b/pillar/config.py new file mode 100644 index 00000000..95301591 --- /dev/null +++ b/pillar/config.py @@ -0,0 +1,66 @@ +import os.path +from collections import defaultdict + +RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' + +SCHEME = 'http' +STORAGE_DIR = '/data/storage/pillar' +SHARED_DIR = '/data/storage/shared' +PORT = 5000 +HOST = '0.0.0.0' +DEBUG = False + +# Authentication settings +BLENDER_ID_ENDPOINT = 'http://blender_id:8000/' + +CDN_USE_URL_SIGNING = True +CDN_SERVICE_DOMAIN_PROTOCOL = 'https' +CDN_SERVICE_DOMAIN = 'test-blendercloud.r.worldssl.net' +CDN_CONTENT_SUBFOLDER = '' +CDN_URL_SIGNING_KEY = '-SECRET-' + +CDN_STORAGE_USER = '-SECRET' +CDN_STORAGE_ADDRESS = 'push-11.cdnsun.com' +CDN_SYNC_LOGS = '/data/storage/logs' +CDN_RSA_KEY = '/data/config/cdnsun_id_rsa' +CDN_KNOWN_HOSTS = '/data/config/known_hosts' + +UPLOADS_LOCAL_STORAGE_THUMBNAILS = { + 's': {'size': (90, 90), 'crop': True}, + 'b': {'size': (160, 160), 'crop': True}, + 't': {'size': (160, 160), 'crop': False}, + 'm': {'size': (320, 320), 'crop': False}, + 'l': {'size': (1024, 1024), 'crop': False}, + 'h': {'size': (2048, 2048), 'crop': False} +} + +BIN_FFPROBE = '/usr/bin/ffprobe' +BIN_FFMPEG = '/usr/bin/ffmpeg' +BIN_SSH = '/usr/bin/ssh' +BIN_RSYNC = '/usr/bin/rsync' + +GCLOUD_APP_CREDENTIALS = os.path.join(os.path.dirname(__file__), 'google_app.json') +GCLOUD_PROJECT = 'blender-cloud' + +ADMIN_USER_GROUP = '5596e975ea893b269af85c0e' +SUBSCRIBER_USER_GROUP = '5596e975ea893b269af85c0f' +BUGSNAG_API_KEY = '' + +ALGOLIA_USER = '-SECRET-' +ALGOLIA_API_KEY = '-SECRET-' +ALGOLIA_INDEX_USERS = 'dev_Users' +ALGOLIA_INDEX_NODES = 'dev_Nodes' + +ZENCODER_API_KEY = '-SECRET-' +ZENCODER_NOTIFICATIONS_SECRET = '-SECRET-' +ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/' + +ENCODING_BACKEND = 'zencoder' # local, flamenco + +# Validity period of links, per file storage backend. Expressed in seconds. +# Shouldn't be more than a year, as this isn't supported by HTTP/1.1. +FILE_LINK_VALIDITY = defaultdict( + lambda: 3600 * 24 * 30, # default of 1 month. + gcs=3600 * 23, # 23 hours for Google Cloud Storage. +) + diff --git a/pillar/config.py.example b/pillar/config.py.example deleted file mode 100644 index 2cf353b3..00000000 --- a/pillar/config.py.example +++ /dev/null @@ -1,79 +0,0 @@ -from collections import defaultdict - - -class Development(object): - PORT = 5000 - HOST = '0.0.0.0' - SCHEME = 'http' - DEBUG = True - RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' - BUGSNAG_API_KEY = '' - - # Authentication settings - BLENDER_ID_ENDPOINT = os.environ.get( - 'BLENDER_ID_ENDPOINT', "https://www.blender.org/id").rstrip("/") - - # Settings for storage - STORAGE_DIR = '/data/storage/pillar' - SHARED_DIR = '/data/storage/shared' - USE_X_SENDFILE = False - - # Fill in only if we are going to use a CDN-attached storage solution. - # Currently we use GCS and do not have enough traffic to justify that - CDN_STORAGE_USER = '' - CDN_STORAGE_ADDRESS = '' - CDN_SYNC_LOGS = '' - CDN_RSA_KEY = '' - CDN_KNOWN_HOSTS = '' - - # Credentials to access project on the Google Cloud where Google Cloud - # Storage is enabled (Pillar will automatically create and manage buckets) - GCLOUD_APP_CREDENTIALS = os.environ.get( - 'GOOGLE_APPLICATION_CREDENTIALS', '/data/config/google_app.json') - GCLOUD_PROJECT = os.environ.get('GCLOUD_PROJECT', 'blender-cloud') - - # Fill in only if we plan to sign our urls using a the CDN - CDN_USE_URL_SIGNING = False - CDN_SERVICE_DOMAIN_PROTOCOL = 'https' - CDN_SERVICE_DOMAIN = '' - CDN_CONTENT_SUBFOLDER = '' - CDN_URL_SIGNING_KEY = '' - - # Settings for image processing (good defaults, should not be altered) - UPLOADS_LOCAL_STORAGE_THUMBNAILS = { - 's': {'size': (90, 90), 'crop': True}, - 'b': {'size': (160, 160), 'crop': True}, - 't': {'size': (160, 160), 'crop': False}, - 'm': {'size': (320, 320), 'crop': False}, - 'l': {'size': (1024, 1024), 'crop': False}, - 'h': {'size': (2048, 2048), 'crop': False} - } - - # Settings for encoder (local will run FFMPEG on the server and is discouraged - # for production setups) - ENCODING_BACKEND = 'zencoder' #local, flamenco - - # Zencoder is a production ready encoding solution - ZENCODER_API_KEY = '' - ZENCODER_NOTIFICATIONS_SECRET = '' - ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/' - - BIN_FFPROBE ='/usr/bin/ffprobe' - BIN_FFMPEG = '/usr/bin/ffmpeg' - BIN_SSH = '/usr/bin/ssh' - BIN_RSYNC = '/usr/bin/rsync' - - # Settings for indexing (currently only Algolia is supported) - ALGOLIA_USER = '' - ALGOLIA_API_KEY = '' - ALGOLIA_INDEX_USERS = '' - - # Validity period of links, per file storage backend. Expressed in seconds. - # Shouldn't be more than a year, as this isn't supported by HTTP/1.1. - FILE_LINK_VALIDITY = defaultdict( - lambda: 3600 * 24 * 30, # default of 1 month. - gcs=3600 * 23, # 23 hours for Google Cloud Storage. - ) - - -class Deployment(Development): pass diff --git a/pillar/manage.py b/pillar/manage.py index 35151640..b7d7c6dc 100755 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -32,28 +32,11 @@ MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost') @manager.command def runserver(): - try: - import config - PORT = config.Development.PORT - HOST = config.Development.HOST - DEBUG = config.Development.DEBUG - app.config['STORAGE_DIR'] = config.Development.STORAGE_DIR - except ImportError: - # Default settings - PORT = 5000 - HOST = '0.0.0.0' - DEBUG = True - app.config['STORAGE_DIR'] = '{0}/application/static/storage'.format( - os.path.dirname(os.path.realpath(__file__))) - # Automatic creation of STORAGE_DIR path if it's missing if not os.path.exists(app.config['STORAGE_DIR']): os.makedirs(app.config['STORAGE_DIR']) - app.run( - port=PORT, - host=HOST, - debug=DEBUG) + app.run() def post_item(entry, data): diff --git a/tests/common_test_class.py b/tests/common_test_class.py index a7ac8246..d69d43d0 100644 --- a/tests/common_test_class.py +++ b/tests/common_test_class.py @@ -12,7 +12,6 @@ import httpretty from common_test_data import EXAMPLE_PROJECT, EXAMPLE_FILE -BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001' # nonexistant server, no trailing slash! MY_PATH = os.path.dirname(os.path.abspath(__file__)) TEST_EMAIL_USER = 'koro' @@ -25,14 +24,15 @@ logging.basicConfig( class AbstractPillarTest(TestMinimal): def setUp(self, **kwargs): - settings_file = os.path.join(MY_PATH, 'common_test_settings.py') - kwargs['settings_file'] = settings_file - os.environ['EVE_SETTINGS'] = settings_file + eve_settings_file = os.path.join(MY_PATH, 'common_test_settings.py') + pillar_config_file = os.path.join(MY_PATH, 'config_testing.py') + kwargs['settings_file'] = eve_settings_file + os.environ['EVE_SETTINGS'] = eve_settings_file + os.environ['PILLAR_CONFIG'] = pillar_config_file super(AbstractPillarTest, self).setUp(**kwargs) from application import app - app.config['BLENDER_ID_ENDPOINT'] = BLENDER_ID_ENDPOINT logging.getLogger('application').setLevel(logging.DEBUG) logging.getLogger('werkzeug').setLevel(logging.DEBUG) logging.getLogger('eve').setLevel(logging.DEBUG) @@ -86,7 +86,7 @@ class AbstractPillarTest(TestMinimal): """Sets up HTTPretty to mock unhappy validation flow.""" httpretty.register_uri(httpretty.POST, - '%s/u/validate_token' % BLENDER_ID_ENDPOINT, + '%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'], body=json.dumps( {'data': {'token': 'Token is invalid'}, 'status': 'fail'}), content_type="application/json") @@ -95,7 +95,7 @@ class AbstractPillarTest(TestMinimal): """Sets up HTTPretty to mock happy validation flow.""" httpretty.register_uri(httpretty.POST, - '%s/u/validate_token' % BLENDER_ID_ENDPOINT, + '%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'], body=json.dumps( {'data': {'user': {'email': TEST_EMAIL_ADDRESS, 'id': 5123}}, 'status': 'success'}), diff --git a/tests/config_testing.py b/tests/config_testing.py new file mode 100644 index 00000000..b7fc3494 --- /dev/null +++ b/tests/config_testing.py @@ -0,0 +1,8 @@ +"""Flask configuration file for unit testing.""" + +BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001' # nonexistant server, no trailing slash! + +DEBUG = True +TESTING = True + +CDN_STORAGE_USER = 'u41508580125621'