More flexible, less error-prone configuration system.
WARNING: make a backup copy of your local config.py before pulling this change, as Git will overwrite it without warning. The configuration defaults to deployment settings, allowing overrides. Overrides are read from config_local.py and from the file pointed to by the PILLAR_CONFIG env var.
This commit is contained in:
parent
a1930c63d0
commit
465b145609
4
.gitignore
vendored
4
.gitignore
vendored
@ -5,11 +5,11 @@
|
||||
*.ropeproject*
|
||||
*.swp
|
||||
|
||||
config.py
|
||||
/pillar/config_local.py
|
||||
|
||||
.ropeproject/*
|
||||
|
||||
pillar/application/static/storage/
|
||||
/pillar/application/static/storage/
|
||||
/build
|
||||
/.cache
|
||||
/pillar/pillar.egg-info/
|
||||
|
||||
@ -4,7 +4,8 @@ import json
|
||||
from bson import ObjectId
|
||||
from datetime import datetime
|
||||
import bugsnag
|
||||
from bugsnag.flask import handle_exceptions
|
||||
import bugsnag.flask
|
||||
import bugsnag.handlers
|
||||
from algoliasearch import algoliasearch
|
||||
from zencoder import Zencoder
|
||||
from flask import g
|
||||
@ -89,9 +90,17 @@ settings_path = os.environ.get(
|
||||
'EVE_SETTINGS', '/data/git/pillar/pillar/settings.py')
|
||||
app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth)
|
||||
|
||||
import config
|
||||
|
||||
app.config.from_object(config.Deployment)
|
||||
# Load configuration from three different sources, to make it easy to override
|
||||
# settings with secrets, as well as for development & testing.
|
||||
app_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
app.config.from_pyfile(os.path.join(app_root, 'config.py'), silent=False)
|
||||
app.config.from_pyfile(os.path.join(app_root, 'config_local.py'), silent=True)
|
||||
from_envvar = os.environ.get('PILLAR_CONFIG')
|
||||
if from_envvar:
|
||||
# Don't use from_envvar, as we want different behaviour. If the envvar
|
||||
# is not set, it's fine (i.e. silent=True), but if it is set and the
|
||||
# configfile doesn't exist, it should error out (i.e. silent=False).
|
||||
app.config.from_pyfile(from_envvar, silent=False)
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
@ -105,11 +114,17 @@ log.setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO)
|
||||
if app.config['DEBUG']:
|
||||
log.info('Pillar starting, debug=%s', app.config['DEBUG'])
|
||||
|
||||
bugsnag.configure(
|
||||
api_key=app.config['BUGSNAG_API_KEY'],
|
||||
project_root="/data/git/pillar/pillar",
|
||||
)
|
||||
handle_exceptions(app)
|
||||
# Configure Bugsnag
|
||||
if not app.config.get('TESTING'):
|
||||
bugsnag.configure(
|
||||
api_key=app.config['BUGSNAG_API_KEY'],
|
||||
project_root="/data/git/pillar/pillar",
|
||||
)
|
||||
bugsnag.flask.handle_exceptions(app)
|
||||
|
||||
bs_handler = bugsnag.handlers.BugsnagHandler()
|
||||
bs_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(bs_handler)
|
||||
|
||||
# Google Cloud project
|
||||
try:
|
||||
|
||||
66
pillar/config.py
Normal file
66
pillar/config.py
Normal file
@ -0,0 +1,66 @@
|
||||
import os.path
|
||||
from collections import defaultdict
|
||||
|
||||
RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
|
||||
|
||||
SCHEME = 'http'
|
||||
STORAGE_DIR = '/data/storage/pillar'
|
||||
SHARED_DIR = '/data/storage/shared'
|
||||
PORT = 5000
|
||||
HOST = '0.0.0.0'
|
||||
DEBUG = False
|
||||
|
||||
# Authentication settings
|
||||
BLENDER_ID_ENDPOINT = 'http://blender_id:8000/'
|
||||
|
||||
CDN_USE_URL_SIGNING = True
|
||||
CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
|
||||
CDN_SERVICE_DOMAIN = 'test-blendercloud.r.worldssl.net'
|
||||
CDN_CONTENT_SUBFOLDER = ''
|
||||
CDN_URL_SIGNING_KEY = '-SECRET-'
|
||||
|
||||
CDN_STORAGE_USER = '-SECRET'
|
||||
CDN_STORAGE_ADDRESS = 'push-11.cdnsun.com'
|
||||
CDN_SYNC_LOGS = '/data/storage/logs'
|
||||
CDN_RSA_KEY = '/data/config/cdnsun_id_rsa'
|
||||
CDN_KNOWN_HOSTS = '/data/config/known_hosts'
|
||||
|
||||
UPLOADS_LOCAL_STORAGE_THUMBNAILS = {
|
||||
's': {'size': (90, 90), 'crop': True},
|
||||
'b': {'size': (160, 160), 'crop': True},
|
||||
't': {'size': (160, 160), 'crop': False},
|
||||
'm': {'size': (320, 320), 'crop': False},
|
||||
'l': {'size': (1024, 1024), 'crop': False},
|
||||
'h': {'size': (2048, 2048), 'crop': False}
|
||||
}
|
||||
|
||||
BIN_FFPROBE = '/usr/bin/ffprobe'
|
||||
BIN_FFMPEG = '/usr/bin/ffmpeg'
|
||||
BIN_SSH = '/usr/bin/ssh'
|
||||
BIN_RSYNC = '/usr/bin/rsync'
|
||||
|
||||
GCLOUD_APP_CREDENTIALS = os.path.join(os.path.dirname(__file__), 'google_app.json')
|
||||
GCLOUD_PROJECT = 'blender-cloud'
|
||||
|
||||
ADMIN_USER_GROUP = '5596e975ea893b269af85c0e'
|
||||
SUBSCRIBER_USER_GROUP = '5596e975ea893b269af85c0f'
|
||||
BUGSNAG_API_KEY = ''
|
||||
|
||||
ALGOLIA_USER = '-SECRET-'
|
||||
ALGOLIA_API_KEY = '-SECRET-'
|
||||
ALGOLIA_INDEX_USERS = 'dev_Users'
|
||||
ALGOLIA_INDEX_NODES = 'dev_Nodes'
|
||||
|
||||
ZENCODER_API_KEY = '-SECRET-'
|
||||
ZENCODER_NOTIFICATIONS_SECRET = '-SECRET-'
|
||||
ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'
|
||||
|
||||
ENCODING_BACKEND = 'zencoder' # local, flamenco
|
||||
|
||||
# Validity period of links, per file storage backend. Expressed in seconds.
|
||||
# Shouldn't be more than a year, as this isn't supported by HTTP/1.1.
|
||||
FILE_LINK_VALIDITY = defaultdict(
|
||||
lambda: 3600 * 24 * 30, # default of 1 month.
|
||||
gcs=3600 * 23, # 23 hours for Google Cloud Storage.
|
||||
)
|
||||
|
||||
@ -1,79 +0,0 @@
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
class Development(object):
|
||||
PORT = 5000
|
||||
HOST = '0.0.0.0'
|
||||
SCHEME = 'http'
|
||||
DEBUG = True
|
||||
RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
|
||||
BUGSNAG_API_KEY = ''
|
||||
|
||||
# Authentication settings
|
||||
BLENDER_ID_ENDPOINT = os.environ.get(
|
||||
'BLENDER_ID_ENDPOINT', "https://www.blender.org/id").rstrip("/")
|
||||
|
||||
# Settings for storage
|
||||
STORAGE_DIR = '/data/storage/pillar'
|
||||
SHARED_DIR = '/data/storage/shared'
|
||||
USE_X_SENDFILE = False
|
||||
|
||||
# Fill in only if we are going to use a CDN-attached storage solution.
|
||||
# Currently we use GCS and do not have enough traffic to justify that
|
||||
CDN_STORAGE_USER = ''
|
||||
CDN_STORAGE_ADDRESS = ''
|
||||
CDN_SYNC_LOGS = ''
|
||||
CDN_RSA_KEY = ''
|
||||
CDN_KNOWN_HOSTS = ''
|
||||
|
||||
# Credentials to access project on the Google Cloud where Google Cloud
|
||||
# Storage is enabled (Pillar will automatically create and manage buckets)
|
||||
GCLOUD_APP_CREDENTIALS = os.environ.get(
|
||||
'GOOGLE_APPLICATION_CREDENTIALS', '/data/config/google_app.json')
|
||||
GCLOUD_PROJECT = os.environ.get('GCLOUD_PROJECT', 'blender-cloud')
|
||||
|
||||
# Fill in only if we plan to sign our urls using a the CDN
|
||||
CDN_USE_URL_SIGNING = False
|
||||
CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
|
||||
CDN_SERVICE_DOMAIN = ''
|
||||
CDN_CONTENT_SUBFOLDER = ''
|
||||
CDN_URL_SIGNING_KEY = ''
|
||||
|
||||
# Settings for image processing (good defaults, should not be altered)
|
||||
UPLOADS_LOCAL_STORAGE_THUMBNAILS = {
|
||||
's': {'size': (90, 90), 'crop': True},
|
||||
'b': {'size': (160, 160), 'crop': True},
|
||||
't': {'size': (160, 160), 'crop': False},
|
||||
'm': {'size': (320, 320), 'crop': False},
|
||||
'l': {'size': (1024, 1024), 'crop': False},
|
||||
'h': {'size': (2048, 2048), 'crop': False}
|
||||
}
|
||||
|
||||
# Settings for encoder (local will run FFMPEG on the server and is discouraged
|
||||
# for production setups)
|
||||
ENCODING_BACKEND = 'zencoder' #local, flamenco
|
||||
|
||||
# Zencoder is a production ready encoding solution
|
||||
ZENCODER_API_KEY = ''
|
||||
ZENCODER_NOTIFICATIONS_SECRET = ''
|
||||
ZENCODER_NOTIFICATIONS_URL = 'http://zencoderfetcher/'
|
||||
|
||||
BIN_FFPROBE ='/usr/bin/ffprobe'
|
||||
BIN_FFMPEG = '/usr/bin/ffmpeg'
|
||||
BIN_SSH = '/usr/bin/ssh'
|
||||
BIN_RSYNC = '/usr/bin/rsync'
|
||||
|
||||
# Settings for indexing (currently only Algolia is supported)
|
||||
ALGOLIA_USER = ''
|
||||
ALGOLIA_API_KEY = ''
|
||||
ALGOLIA_INDEX_USERS = ''
|
||||
|
||||
# Validity period of links, per file storage backend. Expressed in seconds.
|
||||
# Shouldn't be more than a year, as this isn't supported by HTTP/1.1.
|
||||
FILE_LINK_VALIDITY = defaultdict(
|
||||
lambda: 3600 * 24 * 30, # default of 1 month.
|
||||
gcs=3600 * 23, # 23 hours for Google Cloud Storage.
|
||||
)
|
||||
|
||||
|
||||
class Deployment(Development): pass
|
||||
@ -32,28 +32,11 @@ MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost')
|
||||
|
||||
@manager.command
|
||||
def runserver():
|
||||
try:
|
||||
import config
|
||||
PORT = config.Development.PORT
|
||||
HOST = config.Development.HOST
|
||||
DEBUG = config.Development.DEBUG
|
||||
app.config['STORAGE_DIR'] = config.Development.STORAGE_DIR
|
||||
except ImportError:
|
||||
# Default settings
|
||||
PORT = 5000
|
||||
HOST = '0.0.0.0'
|
||||
DEBUG = True
|
||||
app.config['STORAGE_DIR'] = '{0}/application/static/storage'.format(
|
||||
os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
# Automatic creation of STORAGE_DIR path if it's missing
|
||||
if not os.path.exists(app.config['STORAGE_DIR']):
|
||||
os.makedirs(app.config['STORAGE_DIR'])
|
||||
|
||||
app.run(
|
||||
port=PORT,
|
||||
host=HOST,
|
||||
debug=DEBUG)
|
||||
app.run()
|
||||
|
||||
|
||||
def post_item(entry, data):
|
||||
|
||||
@ -12,7 +12,6 @@ import httpretty
|
||||
|
||||
from common_test_data import EXAMPLE_PROJECT, EXAMPLE_FILE
|
||||
|
||||
BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001' # nonexistant server, no trailing slash!
|
||||
MY_PATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
TEST_EMAIL_USER = 'koro'
|
||||
@ -25,14 +24,15 @@ logging.basicConfig(
|
||||
|
||||
class AbstractPillarTest(TestMinimal):
|
||||
def setUp(self, **kwargs):
|
||||
settings_file = os.path.join(MY_PATH, 'common_test_settings.py')
|
||||
kwargs['settings_file'] = settings_file
|
||||
os.environ['EVE_SETTINGS'] = settings_file
|
||||
eve_settings_file = os.path.join(MY_PATH, 'common_test_settings.py')
|
||||
pillar_config_file = os.path.join(MY_PATH, 'config_testing.py')
|
||||
kwargs['settings_file'] = eve_settings_file
|
||||
os.environ['EVE_SETTINGS'] = eve_settings_file
|
||||
os.environ['PILLAR_CONFIG'] = pillar_config_file
|
||||
super(AbstractPillarTest, self).setUp(**kwargs)
|
||||
|
||||
from application import app
|
||||
|
||||
app.config['BLENDER_ID_ENDPOINT'] = BLENDER_ID_ENDPOINT
|
||||
logging.getLogger('application').setLevel(logging.DEBUG)
|
||||
logging.getLogger('werkzeug').setLevel(logging.DEBUG)
|
||||
logging.getLogger('eve').setLevel(logging.DEBUG)
|
||||
@ -86,7 +86,7 @@ class AbstractPillarTest(TestMinimal):
|
||||
"""Sets up HTTPretty to mock unhappy validation flow."""
|
||||
|
||||
httpretty.register_uri(httpretty.POST,
|
||||
'%s/u/validate_token' % BLENDER_ID_ENDPOINT,
|
||||
'%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'],
|
||||
body=json.dumps(
|
||||
{'data': {'token': 'Token is invalid'}, 'status': 'fail'}),
|
||||
content_type="application/json")
|
||||
@ -95,7 +95,7 @@ class AbstractPillarTest(TestMinimal):
|
||||
"""Sets up HTTPretty to mock happy validation flow."""
|
||||
|
||||
httpretty.register_uri(httpretty.POST,
|
||||
'%s/u/validate_token' % BLENDER_ID_ENDPOINT,
|
||||
'%s/u/validate_token' % self.app.config['BLENDER_ID_ENDPOINT'],
|
||||
body=json.dumps(
|
||||
{'data': {'user': {'email': TEST_EMAIL_ADDRESS, 'id': 5123}},
|
||||
'status': 'success'}),
|
||||
|
||||
8
tests/config_testing.py
Normal file
8
tests/config_testing.py
Normal file
@ -0,0 +1,8 @@
|
||||
"""Flask configuration file for unit testing."""
|
||||
|
||||
BLENDER_ID_ENDPOINT = 'http://127.0.0.1:8001' # nonexistant server, no trailing slash!
|
||||
|
||||
DEBUG = True
|
||||
TESTING = True
|
||||
|
||||
CDN_STORAGE_USER = 'u41508580125621'
|
||||
Loading…
x
Reference in New Issue
Block a user