Upgraded gcloud package to 0.11.0

pillar/application/__init__.py

@@ -84,7 +84,7 @@ class ValidateCustomFields(Validator):
                 field, "Error validating properties")
 
 # We specify a settings.py file because when running on wsgi we can't detect it
-# automatically. The default path (which works in Docker) can be overriden with
+# automatically. The default path (which works in Docker) can be overridden with
 # an env variable.
 settings_path = os.environ.get(
     'EVE_SETTINGS', '/data/git/pillar/pillar/settings.py')
@@ -105,6 +105,16 @@ bugsnag.configure(
 )
 handle_exceptions(app)
 
+# Storage backend (GCS)
+try:
+    os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = \
+        app.config['GOOGLE_APPLICATION_CREDENTIALS']
+except KeyError:
+    log.debug("The GOOGLE_APPLICATION_CREDENTIALS configuration value should "
+              "point to an existing and valid JSON file.")
+    raise
+
+
 # Algolia search
 if 'ALGOLIA_USER' in app.config:
     client = algoliasearch.Client(
@@ -122,14 +132,14 @@ if app.config['ENCODING_BACKEND'] == 'zencoder':
 else:
     encoding_service_client = None
 
-from application.utils.authentication import validate_token
+from utils.authentication import validate_token
-from application.utils.authorization import check_permissions
+from utils.authorization import check_permissions
-from application.utils.gcs import update_file_name
+from utils.gcs import update_file_name
-from application.utils.algolia import algolia_index_user_save
+from utils.algolia import algolia_index_user_save
-from application.utils.algolia import algolia_index_node_save
+from utils.algolia import algolia_index_node_save
-from application.utils.activities import activity_subscribe
+from utils.activities import activity_subscribe
-from application.utils.activities import activity_object_add
+from utils.activities import activity_object_add
-from application.utils.activities import notification_parse
+from utils.activities import notification_parse
 from modules.file_storage import process_file
 from modules.file_storage import delete_file
 from modules.file_storage import generate_link

pillar/application/modules/file_storage.py

@@ -1,11 +1,9 @@
 import logging
 import os
-import json
 from multiprocessing import Process
 from bson import ObjectId
 from flask import request
 from flask import Blueprint
-from flask import abort
 from flask import jsonify
 from flask import send_from_directory
 from flask import url_for, helpers
@@ -14,7 +12,6 @@ from application import app
 from application.utils.imaging import generate_local_thumbnails
 from application.utils.imaging import get_video_data
 from application.utils.imaging import ffmpeg_encode
-from application.utils.storage import remote_storage_sync
 from application.utils.storage import push_to_storage
 from application.utils.cdn import hash_file_path
 from application.utils.gcs import GoogleCloudStorageBucket
@@ -24,8 +21,8 @@ from application.utils.encoding import Encoder
 log = logging.getLogger(__name__)
 
 file_storage = Blueprint('file_storage', __name__,
-                        template_folder='templates',
+                         template_folder='templates',
-                        static_folder='../../static/storage',)
+                         static_folder='../../static/storage',)
 
 
 @file_storage.route('/gcs/<bucket_name>/<subdir>/')
@@ -70,13 +67,14 @@ def build_thumbnails(file_path=None, file_id=None):
         file_ = files_collection.find_one({"_id": ObjectId(file_id)})
         file_path = file_['name']
 
-    file_full_path = os.path.join(app.config['SHARED_DIR'], file_path[:2], file_path)
+    file_full_path = os.path.join(app.config['SHARED_DIR'], file_path[:2],
+                                  file_path)
     # Does the original file exist?
     if not os.path.isfile(file_full_path):
         return "", 404
     else:
         thumbnails = generate_local_thumbnails(file_full_path,
-            return_image_stats=True)
+                                               return_image_stats=True)
 
     file_variations = []
     for size, thumbnail in thumbnails.iteritems():
@@ -119,7 +117,8 @@ def index(file_name=None):
     # Sanitize the filename; source: http://stackoverflow.com/questions/7406102/
     file_name = request.form['name']
     keepcharacters = {' ', '.', '_'}
-    file_name = ''.join(c for c in file_name if c.isalnum() or c in keepcharacters).strip()
+    file_name = ''.join(
+        c for c in file_name if c.isalnum() or c in keepcharacters).strip()
     file_name = file_name.lstrip('.')
 
     # Determine & create storage directory

pillar/application/utils/gcs.py

@@ -5,14 +5,13 @@ import bugsnag
 from bson import ObjectId
 from gcloud.storage.client import Client
 from gcloud.exceptions import NotFound
-from oauth2client.client import SignedJwtAssertionCredentials
 from application import app
 
 
 class GoogleCloudStorageBucket(object):
     """Cloud Storage bucket interface. We create a bucket for every project. In
     the bucket we create first level subdirs as follows:
-    - '_' (will contain hashed assets, and stays on top of defaul listing)
+    - '_' (will contain hashed assets, and stays on top of default listing)
     - 'svn' (svn checkout mirror)
     - 'shared' (any additional folder of static folder that is accessed via a
       node of 'storage' node_type)
@@ -21,31 +20,12 @@ class GoogleCloudStorageBucket(object):
     :param bucket_name: Name of the bucket.
 
     :type subdir: string
-    :param subdir: The local entrypoint to browse the bucket.
+    :param subdir: The local entry point to browse the bucket.
 
     """
-    CGS_PROJECT_NAME = app.config['CGS_PROJECT_NAME']
-    GCS_CLIENT_EMAIL = app.config['GCS_CLIENT_EMAIL']
-    GCS_PRIVATE_KEY_PEM = app.config['GCS_PRIVATE_KEY_PEM']
-    GCS_PRIVATE_KEY_P12 = app.config['GCS_PRIVATE_KEY_P12']
-
-    # Load private key in pem format (used by the API)
-    with open(GCS_PRIVATE_KEY_PEM) as f:
-      private_key_pem = f.read()
-    credentials_pem = SignedJwtAssertionCredentials(GCS_CLIENT_EMAIL,
-        private_key_pem,
-        'https://www.googleapis.com/auth/devstorage.full_control')
-
-    # Load private key in p12 format (used by the singed urls generator)
-    with open(GCS_PRIVATE_KEY_P12) as f:
-      private_key_pkcs12 = f.read()
-    credentials_p12 = SignedJwtAssertionCredentials(GCS_CLIENT_EMAIL,
-        private_key_pkcs12,
-        'https://www.googleapis.com/auth/devstorage.full_control')
-
 
     def __init__(self, bucket_name, subdir='_/'):
-        gcs = Client(project=self.CGS_PROJECT_NAME, credentials=self.credentials_pem)
+        gcs = Client()
         self.bucket = gcs.get_bucket(bucket_name)
         self.subdir = subdir
 
@@ -62,7 +42,7 @@ class GoogleCloudStorageBucket(object):
 
         fields_to_return = 'nextPageToken,items(name,size,contentType),prefixes'
         req = self.bucket.list_blobs(fields=fields_to_return, prefix=prefix,
-            delimiter='/')
+                                     delimiter='/')
 
         files = []
         for f in req:
@@ -86,12 +66,11 @@ class GoogleCloudStorageBucket(object):
         list_dict = dict(
             name=os.path.basename(os.path.normpath(path)),
             type='group_storage',
-            children = files + directories
+            children=files + directories
             )
 
         return list_dict
 
-
     def blob_to_dict(self, blob):
         blob.reload()
         expiration = datetime.datetime.now() + datetime.timedelta(days=1)
@@ -101,16 +80,16 @@ class GoogleCloudStorageBucket(object):
             name=os.path.basename(blob.name),
             size=blob.size,
             content_type=blob.content_type,
-            signed_url=blob.generate_signed_url(
+            signed_url=blob.generate_signed_url(expiration),
-                expiration, credentials=self.credentials_p12),
             public_url=blob.public_url)
 
-
     def Get(self, path, to_dict=True):
         """Get selected file info if the path matches.
 
         :type path: string
         :param path: The relative path to the file.
+        :type to_dict: bool
+        :param to_dict: Return the object as a dictionary.
         """
         path = os.path.join(self.subdir, path)
         blob = self.bucket.blob(path)
@@ -122,7 +101,6 @@ class GoogleCloudStorageBucket(object):
         else:
             return None
 
-
     def Post(self, full_path, path=None):
         """Create new blob and upload data to it.
         """
@@ -134,7 +112,6 @@ class GoogleCloudStorageBucket(object):
         return blob
         # return self.blob_to_dict(blob) # Has issues with threading
 
-
     def Delete(self, path):
         """Delete blob (when removing an asset or replacing a preview)"""
 
@@ -146,7 +123,6 @@ class GoogleCloudStorageBucket(object):
         except NotFound:
             return None
 
-
     def update_name(self, blob, name):
         """Set the ContentDisposition metadata so that when a file is downloaded
         it has a human-readable name.

pillar/config.py.example

@@ -25,10 +25,8 @@ class Development(object):
 
     # Credentials to access project on the Google Cloud where Google Cloud
     # Storage is enabled (Pillar will automatically create and manage buckets)
-    GCS_CLIENT_EMAIL = ''
+    GOOGLE_APPLICATION_CREDENTIALS = os.environ.get(
-    GCS_PRIVATE_KEY_P12 = ''
+        'GOOGLE_APPLICATION_CREDENTIALS', '/data/config/google_app.json')
-    GCS_PRIVATE_KEY_PEM = ''
-    CGS_PROJECT_NAME = ''
 
     # Fill in only if we plan to sign our urls using a the CDN
     CDN_USE_URL_SIGNING = False

requirements.txt

@@ -4,29 +4,21 @@ bugsnag==2.3.1
 Cerberus==0.9.1
 Eve==0.5.3
 Events==0.2.1
-Flask==0.10.1
-Flask-PyMongo==0.3.1
 Flask-Script==2.0.5
 flup==1.0.2
-gcloud==0.7.1
+gcloud==0.11.0
 google-apitools==0.4.11
 httplib2==0.9.2
 idna==2.0
 MarkupSafe==0.23
 ndg-httpsclient==0.4.0
-oauth2client==1.5.1
 Pillow==2.8.1
-protobuf==3.0.0a1
-protorpc==0.11.1
 pycparser==2.14
 pycrypto==2.6.1
-pymongo==2.9.1
 pyOpenSSL==0.15.1
 requests==2.9.1
 rsa==3.3
 simplejson==3.8.1
-six==1.10.0
 WebOb==1.5.0
-Werkzeug==0.10.4
 wheel==0.24.0
 zencoder==0.6.5