Implement rename-after-Zencoder-is-done using our Storage API.

This means it's no longer GCS-specific, and can be tested using the local
storage implementation.

Required implementation of a rename operation. To mirror Google's API, I've
implemented the renaming of a Blob as a function on the Bucket class.
To me this makes sense, as it requires creating a new Blob instance, which
shouldn't be done by another Blob.

pillar/api/encoding.py

@@ -10,8 +10,7 @@ from flask import current_app
 from flask import request
 
 from pillar.api import utils
-from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket
-from pillar.api.utils import skip_when_testing
+from pillar.api.file_storage_backends import Bucket
 
 encoding = Blueprint('encoding', __name__)
 log = logging.getLogger(__name__)
@@ -45,14 +44,6 @@ def size_descriptor(width, height):
     return '%ip' % height
 
 
-@skip_when_testing
-def rename_on_gcs(bucket_name, from_path, to_path):
-    gcs = GoogleCloudStorageBucket(str(bucket_name))
-    # FIXME: don't use internal property, but use our bucket/blob API.
-    blob = gcs._gcs_bucket.blob(from_path)
-    gcs._gcs_bucket.rename_blob(blob, to_path)
-
-
 @encoding.route('/zencoder/notifications', methods=['POST'])
 def zencoder_notifications():
     """
@@ -119,7 +110,11 @@ def zencoder_notifications():
              job_state)
 
     # For every variation encoded, try to update the file object
-    root, _ = os.path.splitext(file_doc['file_path'])
+    storage_name, _ = os.path.splitext(file_doc['file_path'])
+    nice_name, _ = os.path.splitext(file_doc['filename'])
+
+    bucket_class = Bucket.for_backend(file_doc['backend'])
+    bucket = bucket_class(str(file_doc['project']))
 
     for output in data['outputs']:
         video_format = output['format']
@@ -140,16 +135,16 @@ def zencoder_notifications():
 
         # Rename the file to include the now-known size descriptor.
         size = size_descriptor(output['width'], output['height'])
-        new_fname = '{}-{}.{}'.format(root, size, video_format)
+        new_fname = f'{storage_name}-{size}.{video_format}'
 
-        # Rename on Google Cloud Storage
+        # Rename the file on the storage.
+        blob = bucket.blob(variation['file_path'])
         try:
-            rename_on_gcs(file_doc['project'],
-                          '_/' + variation['file_path'],
-                          '_/' + new_fname)
+            new_blob = bucket.rename_blob(blob, new_fname)
+            new_blob.update_filename(f'{nice_name}-{size}.{video_format}')
         except Exception:
-            log.warning('Unable to rename GCS blob %r to %r. Keeping old name.',
-                        variation['file_path'], new_fname, exc_info=True)
+            log.warning('Unable to rename blob %r to %r. Keeping old name.',
+                        blob, new_fname, exc_info=True)
         else:
             variation['file_path'] = new_fname
 

pillar/api/file_storage_backends/abstract.py

@@ -67,6 +67,10 @@ class Bucket(metaclass=abc.ABCMeta):
         same storage backend.
         """
 
+    @abc.abstractmethod
+    def rename_blob(self, blob: 'Blob', new_name: str) -> 'Blob':
+        """Rename the blob, returning the new Blob."""
+
     @classmethod
     def copy_to_bucket(cls, blob_name, src_project_id: ObjectId, dest_project_id: ObjectId):
         """Copies a file from one bucket to the other."""

pillar/api/file_storage_backends/gcs.py

@@ -10,7 +10,7 @@ import gcloud.exceptions as gcloud_exc
 from flask import current_app, g
 from werkzeug.local import LocalProxy
 
-from .abstract import Bucket, Blob, Path, FileType
+from .abstract import Bucket, Blob, FileType
 
 log = logging.getLogger(__name__)
 
@@ -132,15 +132,28 @@ class GoogleCloudStorageBucket(Bucket):
 
         return self._gcs_bucket.copy_blob(blob.gblob, to_bucket._gcs_bucket)
 
+    def rename_blob(self, blob: 'GoogleCloudStorageBlob', new_name: str) \
+            -> 'GoogleCloudStorageBlob':
+        """Rename the blob, returning the new Blob."""
+
+        assert isinstance(blob, GoogleCloudStorageBlob)
+
+        new_name = os.path.join(self.subdir, new_name)
+
+        self._log.info('Renaming %s to %r', blob, new_name)
+        new_gblob = self._gcs_bucket.rename_blob(blob.gblob, new_name)
+        return GoogleCloudStorageBlob(new_gblob.name, self, gblob=new_gblob)
+
 
 class GoogleCloudStorageBlob(Blob):
     """GCS blob interface."""
 
-    def __init__(self, name: str, bucket: GoogleCloudStorageBucket) -> None:
+    def __init__(self, name: str, bucket: GoogleCloudStorageBucket,
+                 *, gblob: gcloud.storage.blob.Blob=None) -> None:
         super().__init__(name, bucket)
 
         self._log = logging.getLogger(f'{__name__}.GoogleCloudStorageBlob')
-        self.gblob = bucket._gcs_get(name, chunk_size=256 * 1024 * 2)
+        self.gblob = gblob or bucket._gcs_get(name, chunk_size=256 * 1024 * 2)
 
     def create_from_file(self, file_obj: FileType, *,
                          content_type: str,

pillar/api/file_storage_backends/local.py

@@ -52,6 +52,21 @@ class LocalBucket(Bucket):
         with open(blob.abspath(), 'rb') as src_file:
             dest_blob.create_from_file(src_file, content_type='application/x-octet-stream')
 
+    def rename_blob(self, blob: 'LocalBlob', new_name: str) -> 'LocalBlob':
+        """Rename the blob, returning the new Blob."""
+
+        assert isinstance(blob, LocalBlob)
+
+        self._log.info('Renaming %s to %r', blob, new_name)
+        new_blob = LocalBlob(new_name, self)
+
+        old_path = blob.abspath()
+        new_path = new_blob.abspath()
+        new_path.parent.mkdir(parents=True, exist_ok=True)
+        old_path.rename(new_path)
+
+        return new_blob
+
 
 class LocalBlob(Blob):
     """Blob representing a local file on the filesystem."""
@@ -103,3 +118,9 @@ class LocalBlob(Blob):
 
     def exists(self) -> bool:
         return self.abspath().exists()
+
+    def touch(self):
+        """Touch the file, creating parent directories if needed."""
+        path = self.abspath()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.touch(exist_ok=True)