diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 4885ad1e..8b67bc80 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -84,7 +84,7 @@ class ValidateCustomFields(Validator): field, "Error validating properties") # We specify a settings.py file because when running on wsgi we can't detect it -# automatically. The default path (which works in Docker) can be overriden with +# automatically. The default path (which works in Docker) can be overridden with # an env variable. settings_path = os.environ.get( 'EVE_SETTINGS', '/data/git/pillar/pillar/settings.py') @@ -105,6 +105,16 @@ bugsnag.configure( ) handle_exceptions(app) +# Storage backend (GCS) +try: + os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = \ + app.config['GOOGLE_APPLICATION_CREDENTIALS'] +except KeyError: + log.debug("The GOOGLE_APPLICATION_CREDENTIALS configuration value should " + "point to an existing and valid JSON file.") + raise + + # Algolia search if 'ALGOLIA_USER' in app.config: client = algoliasearch.Client( @@ -122,14 +132,14 @@ if app.config['ENCODING_BACKEND'] == 'zencoder': else: encoding_service_client = None -from application.utils.authentication import validate_token -from application.utils.authorization import check_permissions -from application.utils.gcs import update_file_name -from application.utils.algolia import algolia_index_user_save -from application.utils.algolia import algolia_index_node_save -from application.utils.activities import activity_subscribe -from application.utils.activities import activity_object_add -from application.utils.activities import notification_parse +from utils.authentication import validate_token +from utils.authorization import check_permissions +from utils.gcs import update_file_name +from utils.algolia import algolia_index_user_save +from utils.algolia import algolia_index_node_save +from utils.activities import activity_subscribe +from utils.activities import activity_object_add +from utils.activities import notification_parse from modules.file_storage import process_file from modules.file_storage import delete_file from modules.file_storage import generate_link diff --git a/pillar/application/modules/file_storage.py b/pillar/application/modules/file_storage.py index 9f2d6dc1..65c39ac2 100644 --- a/pillar/application/modules/file_storage.py +++ b/pillar/application/modules/file_storage.py @@ -1,11 +1,9 @@ import logging import os -import json from multiprocessing import Process from bson import ObjectId from flask import request from flask import Blueprint -from flask import abort from flask import jsonify from flask import send_from_directory from flask import url_for, helpers @@ -14,7 +12,6 @@ from application import app from application.utils.imaging import generate_local_thumbnails from application.utils.imaging import get_video_data from application.utils.imaging import ffmpeg_encode -from application.utils.storage import remote_storage_sync from application.utils.storage import push_to_storage from application.utils.cdn import hash_file_path from application.utils.gcs import GoogleCloudStorageBucket @@ -24,8 +21,8 @@ from application.utils.encoding import Encoder log = logging.getLogger(__name__) file_storage = Blueprint('file_storage', __name__, - template_folder='templates', - static_folder='../../static/storage',) + template_folder='templates', + static_folder='../../static/storage',) @file_storage.route('/gcs///') @@ -70,13 +67,14 @@ def build_thumbnails(file_path=None, file_id=None): file_ = files_collection.find_one({"_id": ObjectId(file_id)}) file_path = file_['name'] - file_full_path = os.path.join(app.config['SHARED_DIR'], file_path[:2], file_path) + file_full_path = os.path.join(app.config['SHARED_DIR'], file_path[:2], + file_path) # Does the original file exist? if not os.path.isfile(file_full_path): return "", 404 else: thumbnails = generate_local_thumbnails(file_full_path, - return_image_stats=True) + return_image_stats=True) file_variations = [] for size, thumbnail in thumbnails.iteritems(): @@ -119,7 +117,8 @@ def index(file_name=None): # Sanitize the filename; source: http://stackoverflow.com/questions/7406102/ file_name = request.form['name'] keepcharacters = {' ', '.', '_'} - file_name = ''.join(c for c in file_name if c.isalnum() or c in keepcharacters).strip() + file_name = ''.join( + c for c in file_name if c.isalnum() or c in keepcharacters).strip() file_name = file_name.lstrip('.') # Determine & create storage directory diff --git a/pillar/application/utils/gcs.py b/pillar/application/utils/gcs.py index 5c8a7fe6..09487d59 100644 --- a/pillar/application/utils/gcs.py +++ b/pillar/application/utils/gcs.py @@ -5,14 +5,13 @@ import bugsnag from bson import ObjectId from gcloud.storage.client import Client from gcloud.exceptions import NotFound -from oauth2client.client import SignedJwtAssertionCredentials from application import app class GoogleCloudStorageBucket(object): """Cloud Storage bucket interface. We create a bucket for every project. In the bucket we create first level subdirs as follows: - - '_' (will contain hashed assets, and stays on top of defaul listing) + - '_' (will contain hashed assets, and stays on top of default listing) - 'svn' (svn checkout mirror) - 'shared' (any additional folder of static folder that is accessed via a node of 'storage' node_type) @@ -21,31 +20,12 @@ class GoogleCloudStorageBucket(object): :param bucket_name: Name of the bucket. :type subdir: string - :param subdir: The local entrypoint to browse the bucket. + :param subdir: The local entry point to browse the bucket. """ - CGS_PROJECT_NAME = app.config['CGS_PROJECT_NAME'] - GCS_CLIENT_EMAIL = app.config['GCS_CLIENT_EMAIL'] - GCS_PRIVATE_KEY_PEM = app.config['GCS_PRIVATE_KEY_PEM'] - GCS_PRIVATE_KEY_P12 = app.config['GCS_PRIVATE_KEY_P12'] - - # Load private key in pem format (used by the API) - with open(GCS_PRIVATE_KEY_PEM) as f: - private_key_pem = f.read() - credentials_pem = SignedJwtAssertionCredentials(GCS_CLIENT_EMAIL, - private_key_pem, - 'https://www.googleapis.com/auth/devstorage.full_control') - - # Load private key in p12 format (used by the singed urls generator) - with open(GCS_PRIVATE_KEY_P12) as f: - private_key_pkcs12 = f.read() - credentials_p12 = SignedJwtAssertionCredentials(GCS_CLIENT_EMAIL, - private_key_pkcs12, - 'https://www.googleapis.com/auth/devstorage.full_control') - def __init__(self, bucket_name, subdir='_/'): - gcs = Client(project=self.CGS_PROJECT_NAME, credentials=self.credentials_pem) + gcs = Client() self.bucket = gcs.get_bucket(bucket_name) self.subdir = subdir @@ -62,7 +42,7 @@ class GoogleCloudStorageBucket(object): fields_to_return = 'nextPageToken,items(name,size,contentType),prefixes' req = self.bucket.list_blobs(fields=fields_to_return, prefix=prefix, - delimiter='/') + delimiter='/') files = [] for f in req: @@ -86,12 +66,11 @@ class GoogleCloudStorageBucket(object): list_dict = dict( name=os.path.basename(os.path.normpath(path)), type='group_storage', - children = files + directories + children=files + directories ) return list_dict - def blob_to_dict(self, blob): blob.reload() expiration = datetime.datetime.now() + datetime.timedelta(days=1) @@ -101,16 +80,16 @@ class GoogleCloudStorageBucket(object): name=os.path.basename(blob.name), size=blob.size, content_type=blob.content_type, - signed_url=blob.generate_signed_url( - expiration, credentials=self.credentials_p12), + signed_url=blob.generate_signed_url(expiration), public_url=blob.public_url) - def Get(self, path, to_dict=True): """Get selected file info if the path matches. :type path: string :param path: The relative path to the file. + :type to_dict: bool + :param to_dict: Return the object as a dictionary. """ path = os.path.join(self.subdir, path) blob = self.bucket.blob(path) @@ -122,7 +101,6 @@ class GoogleCloudStorageBucket(object): else: return None - def Post(self, full_path, path=None): """Create new blob and upload data to it. """ @@ -134,7 +112,6 @@ class GoogleCloudStorageBucket(object): return blob # return self.blob_to_dict(blob) # Has issues with threading - def Delete(self, path): """Delete blob (when removing an asset or replacing a preview)""" @@ -146,7 +123,6 @@ class GoogleCloudStorageBucket(object): except NotFound: return None - def update_name(self, blob, name): """Set the ContentDisposition metadata so that when a file is downloaded it has a human-readable name. diff --git a/pillar/config.py.example b/pillar/config.py.example index 938b24c6..e37ab90f 100644 --- a/pillar/config.py.example +++ b/pillar/config.py.example @@ -25,10 +25,8 @@ class Development(object): # Credentials to access project on the Google Cloud where Google Cloud # Storage is enabled (Pillar will automatically create and manage buckets) - GCS_CLIENT_EMAIL = '' - GCS_PRIVATE_KEY_P12 = '' - GCS_PRIVATE_KEY_PEM = '' - CGS_PROJECT_NAME = '' + GOOGLE_APPLICATION_CREDENTIALS = os.environ.get( + 'GOOGLE_APPLICATION_CREDENTIALS', '/data/config/google_app.json') # Fill in only if we plan to sign our urls using a the CDN CDN_USE_URL_SIGNING = False diff --git a/requirements.txt b/requirements.txt index 9ba8f929..40b2d11d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,29 +4,21 @@ bugsnag==2.3.1 Cerberus==0.9.1 Eve==0.5.3 Events==0.2.1 -Flask==0.10.1 -Flask-PyMongo==0.3.1 Flask-Script==2.0.5 flup==1.0.2 -gcloud==0.7.1 +gcloud==0.11.0 google-apitools==0.4.11 httplib2==0.9.2 idna==2.0 MarkupSafe==0.23 ndg-httpsclient==0.4.0 -oauth2client==1.5.1 Pillow==2.8.1 -protobuf==3.0.0a1 -protorpc==0.11.1 pycparser==2.14 pycrypto==2.6.1 -pymongo==2.9.1 pyOpenSSL==0.15.1 requests==2.9.1 rsa==3.3 simplejson==3.8.1 -six==1.10.0 WebOb==1.5.0 -Werkzeug==0.10.4 wheel==0.24.0 zencoder==0.6.5