From 4d6bf65a996fec3c43e3d766b93f95083c3b8846 Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Wed, 9 Nov 2016 02:14:45 +0100 Subject: [PATCH] Attempt at proper naming Using Bucket and Blob as base classes. --- pillar/api/encoding.py | 4 +- pillar/api/file_storage/__init__.py | 29 ++++----- pillar/api/projects/hooks.py | 2 +- pillar/api/utils/gcs.py | 46 +++++++-------- pillar/api/utils/storage.py | 91 ++++++++++++++++++++--------- 5 files changed, 107 insertions(+), 65 deletions(-) diff --git a/pillar/api/encoding.py b/pillar/api/encoding.py index a33bf795..138bcbf4 100644 --- a/pillar/api/encoding.py +++ b/pillar/api/encoding.py @@ -45,8 +45,8 @@ def size_descriptor(width, height): @skip_when_testing def rename_on_gcs(bucket_name, from_path, to_path): gcs = GoogleCloudStorageBucket(str(bucket_name)) - blob = gcs.bucket.blob(from_path) - gcs.bucket.rename_blob(blob, to_path) + blob = gcs.gcs_bucket.blob(from_path) + gcs.gcs_bucket.rename_blob(blob, to_path) @encoding.route('/zencoder/notifications', methods=['POST']) diff --git a/pillar/api/file_storage/__init__.py b/pillar/api/file_storage/__init__.py index 636cd8ef..a087929f 100644 --- a/pillar/api/file_storage/__init__.py +++ b/pillar/api/file_storage/__init__.py @@ -28,7 +28,7 @@ from pillar.api.utils.cdn import hash_file_path from pillar.api.utils.encoding import Encoder from pillar.api.utils.gcs import GoogleCloudStorageBucket, \ GoogleCloudStorageBlob -from pillar.api.utils.storage import PillarStorage, PillarStorageFile +from pillar.api.utils.storage import LocalBucket, LocalBlob, default_storage_backend log = logging.getLogger(__name__) @@ -317,7 +317,7 @@ def generate_link(backend, file_path, project_id=None, is_public=False): return blob['public_url'] return blob['signed_url'] if backend == 'local': - f = PillarStorageFile(project_id, file_path) + f = LocalBlob(project_id, file_path) return url_for('file_storage.index', file_name=f.partial_path, _external=True, _scheme=current_app.config['SCHEME']) if backend == 'pillar': @@ -705,14 +705,17 @@ def stream_to_storage(project_id): # Fake a Blob object. file_in_storage = type('Blob', (), {'size': file_size}) else: - if current_app.config['STORAGE_BACKEND'] == 'gcs': - file_in_storage, storage_backend = stream_to_gcs( - file_id, file_size, internal_fname, project_id, stream_for_gcs, - uploaded_file.mimetype) - elif current_app.config['STORAGE_BACKEND'] == 'local': - storage_backend = PillarStorage(project_id) - file_in_storage = PillarStorageFile(project_id, internal_fname) - file_in_storage.create_from_file(stream_for_gcs, file_size) + bucket = default_storage_backend(project_id) + blob = bucket.blob(internal_fname) + blob.create_from_file(stream_for_gcs, file_size) + # if current_app.config['STORAGE_BACKEND'] == 'gcs': + # file_in_storage, storage_backend = stream_to_gcs( + # file_id, file_size, internal_fname, project_id, + # stream_for_gcs, uploaded_file.mimetype) + # elif current_app.config['STORAGE_BACKEND'] == 'local': + # storage_backend = LocalBucket(project_id) + # file_in_storage = LocalBlob(project_id, internal_fname) + # file_in_storage.create_from_file(stream_for_gcs, file_size) log.debug('Marking uploaded file id=%s, fname=%s, ' 'size=%i as "queued_for_processing"', @@ -720,11 +723,11 @@ def stream_to_storage(project_id): update_file_doc(file_id, status='queued_for_processing', file_path=internal_fname, - length=file_in_storage.size, + length=blob.size, content_type=uploaded_file.mimetype) log.debug('Processing uploaded file id=%s, fname=%s, size=%i', file_id, - internal_fname, file_in_storage.size) + internal_fname, blob.size) process_file(storage_backend, file_id, local_file) # Local processing is done, we can close the local file so it is removed. @@ -732,7 +735,7 @@ def stream_to_storage(project_id): local_file.close() log.debug('Handled uploaded file id=%s, fname=%s, size=%i, status=%i', - file_id, internal_fname, file_in_storage.size, status) + file_id, internal_fname, blob.size, status) # Status is 200 if the file already existed, and 201 if it was newly # created. diff --git a/pillar/api/projects/hooks.py b/pillar/api/projects/hooks.py index e6e1d3ce..cf91ce91 100644 --- a/pillar/api/projects/hooks.py +++ b/pillar/api/projects/hooks.py @@ -172,7 +172,7 @@ def after_inserting_project(project, db_user): else: try: gcs_storage = GoogleCloudStorageBucket(str(project_id)) - if gcs_storage.bucket.exists(): + if gcs_storage.gcs_bucket.exists(): log.info('Created GCS instance for project %s', project_id) else: log.warning('Unable to create GCS instance for project %s', project_id) diff --git a/pillar/api/utils/gcs.py b/pillar/api/utils/gcs.py index e6c1cc10..620ce43a 100644 --- a/pillar/api/utils/gcs.py +++ b/pillar/api/utils/gcs.py @@ -9,7 +9,7 @@ from gcloud.exceptions import NotFound from flask import current_app, g from werkzeug.local import LocalProxy -from pillar.api.utils.storage import StorageBackend, FileInStorage +from pillar.api.utils.storage import register_backend, Bucket, Blob log = logging.getLogger(__name__) @@ -34,7 +34,8 @@ def get_client(): gcs = LocalProxy(get_client) -class GoogleCloudStorageBucket(StorageBackend): +@register_backend('gcs') +class GoogleCloudStorageBucket(Bucket): """Cloud Storage bucket interface. We create a bucket for every project. In the bucket we create first level subdirs as follows: - '_' (will contain hashed assets, and stays on top of default listing) @@ -50,16 +51,16 @@ class GoogleCloudStorageBucket(StorageBackend): """ - def __init__(self, bucket_name, subdir='_/'): - super(GoogleCloudStorageBucket, self).__init__(backend='cgs') + def __init__(self, name, subdir='_/'): + super(GoogleCloudStorageBucket, self).__init__(name=name) try: - self.bucket = gcs.get_bucket(bucket_name) + self.gcs_bucket = gcs.get_bucket(name) except NotFound: - self.bucket = gcs.bucket(bucket_name) + self.gcs_bucket = gcs.bucket(name) # Hardcode the bucket location to EU - self.bucket.location = 'EU' + self.gcs_bucket.location = 'EU' # Optionally enable CORS from * (currently only used for vrview) - # self.bucket.cors = [ + # self.gcs_bucket.cors = [ # { # "origin": ["*"], # "responseHeader": ["Content-Type"], @@ -67,10 +68,13 @@ class GoogleCloudStorageBucket(StorageBackend): # "maxAgeSeconds": 3600 # } # ] - self.bucket.create() + self.gcs_bucket.create() self.subdir = subdir + def blob(self, blob_name): + return GoogleCloudStorageBlob(name=blob_name, bucket=self) + def List(self, path=None): """Display the content of a subdir in the project bucket. If the path points to a file the listing is simply empty. @@ -83,8 +87,8 @@ class GoogleCloudStorageBucket(StorageBackend): prefix = os.path.join(self.subdir, path) fields_to_return = 'nextPageToken,items(name,size,contentType),prefixes' - req = self.bucket.list_blobs(fields=fields_to_return, prefix=prefix, - delimiter='/') + req = self.gcs_bucket.list_blobs(fields=fields_to_return, prefix=prefix, + delimiter='/') files = [] for f in req: @@ -134,7 +138,7 @@ class GoogleCloudStorageBucket(StorageBackend): :param to_dict: Return the object as a dictionary. """ path = os.path.join(self.subdir, path) - blob = self.bucket.blob(path) + blob = self.gcs_bucket.blob(path) if blob.exists(): if to_dict: return self.blob_to_dict(blob) @@ -147,7 +151,7 @@ class GoogleCloudStorageBucket(StorageBackend): """Create new blob and upload data to it. """ path = path if path else os.path.join('_', os.path.basename(full_path)) - blob = self.bucket.blob(path) + blob = self.gcs_bucket.blob(path) if blob.exists(): return None blob.upload_from_filename(full_path) @@ -179,22 +183,18 @@ class GoogleCloudStorageBucket(StorageBackend): """ assert isinstance(to_bucket, GoogleCloudStorageBucket) - return self.bucket.copy_blob(blob, to_bucket.bucket) + return self.gcs_bucket.copy_blob(blob, to_bucket.gcs_bucket) def get_blob(self, internal_fname, chunk_size=256 * 1024 * 2): - return self.bucket.blob('_/' + internal_fname, chunk_size) + return self.gcs_bucket.blob('_/' + internal_fname, chunk_size) -class GoogleCloudStorageBlob(FileInStorage): +class GoogleCloudStorageBlob(Blob): """GCS blob interface.""" - def __init__(self, bucket, internal_fname): - super(GoogleCloudStorageBlob, self).__init__(backend='cgs') + def __init__(self, name, bucket): + super(GoogleCloudStorageBlob, self).__init__(name, bucket) - self.blob = bucket.blob('_/' + internal_fname, chunk_size=256 * 1024 * 2) - self.size = self.get_size() - - def get_size(self): - return self.blob.size + self.blob = bucket.gcs_bucket.blob('_/' + name, chunk_size=256 * 1024 * 2) def update_file_name(node): diff --git a/pillar/api/utils/storage.py b/pillar/api/utils/storage.py index 9e251842..3601bce3 100644 --- a/pillar/api/utils/storage.py +++ b/pillar/api/utils/storage.py @@ -22,52 +22,91 @@ def register_backend(backend_name): return wrapper -class StorageBackend(object): +class Bucket(object): """Can be a GCS bucket or simply a project folder in Pillar - :type backend: string - :param backend: Name of the storage backend (gcs, pillar, cdnsun). + :type name: string + :param name: Name of the bucket. As a convention, we use the ID of + the project to name the bucket. """ __metaclass__ = abc.ABCMeta - def __init__(self, backend): - self.backend = backend + def __init__(self, name): + self.name = name @abc.abstractmethod - def upload_file(self, param1, param2, param3): - """docstuff""" + def blob(self, blob_name): + """Factory constructor for blob object. + + :type blob_name: string + :param blob_name: The name of the blob to be instantiated. + """ + return Blob(name=blob_name, bucket=self) + + @abc.abstractmethod + def get_blob(self, blob_name): + """Get a blob object by name. + + If the blob exists return the object, otherwise None. + """ pass -class FileInStorage(object): +class Blob(object): """A wrapper for file or blob objects. - :type backend: string - :param backend: Name of the storage backend (gcs, pillar, cdnsun). + :type name: string + :param name: Name of the blob. """ - def __init__(self, backend): - self.backend = backend - self.path = None - self.size = None + __metaclass__ = abc.ABCMeta + + def __init__(self, name, bucket): + self.name = name + self.bucket = bucket + self._size_in_bytes = None + + @property + def size(self): + """Size of the object, in bytes. + + :rtype: integer or ``NoneType`` + :returns: The size of the blob or ``None`` if the property + is not set locally. + """ + size = self._size_in_bytes + if size is not None: + return int(size) + return self._size_in_bytes + + @abc.abstractmethod + def create_from_file(self, uploaded_file, file_size): + pass @register_backend('local') -class PillarStorage(StorageBackend): - def __init__(self, project_id): - super(PillarStorage, self).__init__(backend='local') +class LocalBucket(Bucket): + def __init__(self, name): + super(LocalBucket, self).__init__(name=name) + + def blob(self, blob_name): + return LocalBlob(name=blob_name, bucket=self) + + def get_blob(self, blob_name): + # Check if file exists, otherwise None + return None -class PillarStorageFile(FileInStorage): - def __init__(self, project_id, internal_fname): - super(PillarStorageFile, self).__init__(backend='local') +class LocalBlob(Blob): + def __init__(self, name, bucket): + super(LocalBlob, self).__init__(name=name, bucket=bucket) - self.size = None - self.partial_path = os.path.join(project_id[:2], project_id, - internal_fname[:2], internal_fname) + bucket_name = bucket.name + self.partial_path = os.path.join(bucket_name[:2], bucket_name, + name[:2], name) self.path = os.path.join( current_app.config['STORAGE_DIR'], self.partial_path) @@ -78,11 +117,11 @@ class PillarStorageFile(FileInStorage): with open(self.path, 'wb') as outfile: shutil.copyfileobj(uploaded_file, outfile) - self.size = file_size + self._size_in_bytes = file_size -def default_storage_backend(): +def default_storage_backend(name): from flask import current_app backend_cls = backends[current_app.config['STORAGE_BACKEND']] - return backend_cls() + return backend_cls(name)