diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 4b549a17..37695791 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -349,6 +349,7 @@ def post_GET_user(request, payload): app.on_post_GET_users += post_GET_user from modules.file_storage import process_file +from modules.file_storage import delete_file def post_POST_files(request, payload): """After an file object has been created, we do the necessary processing @@ -362,15 +363,15 @@ from utils.cdn import hash_file_path from application.utils.gcs import GoogleCloudStorageBucket # Hook to check the backend of a file resource, to build an appropriate link # that can be used by the client to retrieve the actual file. -def generate_link(backend, path, project_id=None): +def generate_link(backend, file_path, project_id=None): if backend == 'gcs': storage = GoogleCloudStorageBucket(project_id) - blob = storage.Get(path) + blob = storage.Get(file_path) link = blob['signed_url'] elif backend == 'pillar': - link = url_for('file_storage.index', file_name=path, _external=True) + link = url_for('file_storage.index', file_name=file_path, _external=True) elif backend == 'cdnsun': - link = hash_file_path(path, None) + link = hash_file_path(file_path, None) else: link = None return link @@ -378,18 +379,24 @@ def generate_link(backend, path, project_id=None): def before_returning_file(response): # TODO: add project id to all files project_id = None if 'project' not in response else str(response['project']) - response['link'] = generate_link(response['backend'], response['path'], project_id) + response['link'] = generate_link(response['backend'], response['file_path'], project_id) def before_returning_files(response): for item in response['_items']: # TODO: add project id to all files project_id = None if 'project' not in item else str(item['project']) - item['link'] = generate_link(item['backend'], item['path'], project_id) + item['link'] = generate_link(item['backend'], item['file_path'], project_id) app.on_fetched_item_files += before_returning_file app.on_fetched_resource_files += before_returning_files + +def before_deleting_file(item): + delete_file(item) + +app.on_delete_item_files += before_deleting_file + # The file_storage module needs app to be defined from modules.file_storage import file_storage #from modules.file_storage.serve import * diff --git a/pillar/application/modules/file_storage/__init__.py b/pillar/application/modules/file_storage/__init__.py index 84d5d067..a380c48f 100644 --- a/pillar/application/modules/file_storage/__init__.py +++ b/pillar/application/modules/file_storage/__init__.py @@ -51,7 +51,7 @@ def build_thumbnails(file_path=None, file_id=None): files_collection = app.data.driver.db['files'] if file_path: # Search file with backend "pillar" and path=file_path - file_ = files_collection.find({"path": "{0}".format(file_path)}) + file_ = files_collection.find({"file_path": "{0}".format(file_path)}) file_ = file_[0] if file_id: @@ -72,7 +72,7 @@ def build_thumbnails(file_path=None, file_id=None): if thumbnail.get('exists'): # If a thumbnail was already made, we just continue continue - basename = os.path.basename(thumbnail['path']) + basename = os.path.basename(thumbnail['file_path']) root, ext = os.path.splitext(basename) file_object = dict( name=root, @@ -88,7 +88,7 @@ def build_thumbnails(file_path=None, file_id=None): md5=thumbnail['md5'], filename=basename, backend=file_['backend'], - path=basename, + file_path=basename, project=file_['project']) # Commit to database r = post_item('files', file_object) @@ -131,7 +131,7 @@ def process_file(src_file): content_type = src_file['content_type'].split('/') src_file['format'] = content_type[1] mime_type = content_type[0] - src_file['path'] = src_file['name'] + src_file['file_path'] = src_file['name'] if mime_type == 'image': from PIL import Image @@ -183,7 +183,7 @@ def process_file(src_file): md5="", # Available after encode filename=os.path.split(filename)[1], backend=src_file['backend'], - path=filename) + file_path=filename) file_object_id = files_collection.save(file_object) # Append the ObjectId to the new list @@ -226,3 +226,27 @@ def process_file(src_file): file_asset = files_collection.find_and_modify( {'_id': src_file['_id']}, src_file) + + +def delete_file(file_item): + def process_file_delete(file_item): + """Given a file item, delete the actual file from the storage backend. + This function can be probably made self-calling.""" + if file_item['backend'] == 'gcs': + storage = GoogleCloudStorageBucket(str(file_item['project'])) + storage.Delete(file_item['file_path']) + return True + elif backend == 'pillar': + pass + elif backend == 'cdnsun': + pass + else: + pass + files_collection = app.data.driver.db['files'] + # Collect children (variations) of the original file + children = files_collection.find({'parent': file_item['_id']}) + for child in children: + process_file_delete(child) + # Finally remove the original file + process_file_delete(file_item) + diff --git a/pillar/application/utils/gcs.py b/pillar/application/utils/gcs.py index 615587b5..eef1a113 100644 --- a/pillar/application/utils/gcs.py +++ b/pillar/application/utils/gcs.py @@ -2,6 +2,7 @@ import os import time import datetime from gcloud.storage.client import Client +from gcloud.exceptions import NotFound from oauth2client.client import SignedJwtAssertionCredentials from application import app @@ -101,7 +102,7 @@ class GoogleCloudStorageBucket(object): signed_url=blob.generate_signed_url(expiration, credentials=self.credentials_p12)) - def Get(self, path): + def Get(self, path, to_dict=True): """Get selected file info if the path matches. :type path: string @@ -110,7 +111,10 @@ class GoogleCloudStorageBucket(object): path = os.path.join(self.subdir, path) blob = self.bucket.blob(path) if blob.exists(): - return self.blob_to_dict(blob) + if to_dict: + return self.blob_to_dict(blob) + else: + return blob else: return None @@ -125,3 +129,15 @@ class GoogleCloudStorageBucket(object): blob.upload_from_filename(full_path) return blob # return self.blob_to_dict(blob) # Has issues with threading + + + def Delete(self, path): + """Delete blob (when removing an asset or replacing a preview)""" + + # We want to get the actual blob to delete + blob = self.Get(path, to_dict=False) + try: + blob.delete() + return True + except NotFound: + return None diff --git a/pillar/application/utils/imaging.py b/pillar/application/utils/imaging.py index fc1a2968..cdd76908 100644 --- a/pillar/application/utils/imaging.py +++ b/pillar/application/utils/imaging.py @@ -43,7 +43,7 @@ def generate_local_thumbnails(src, return_image_stats=False): format = im.format.lower() # Get format thumbnails[size] = dict( - path=dst, # Full path, to be processed before storage + file_path=dst, # Full path, to be processed before storage length=length, width=width, height=height, diff --git a/pillar/manage.py b/pillar/manage.py index e164112a..adc67f63 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -623,7 +623,6 @@ def populate_node_types(old_ids={}): 'methods': ['GET', 'PUT', 'POST'] }], 'users': [], - 'world': ['GET'] } } @@ -664,7 +663,6 @@ def populate_node_types(old_ids={}): 'methods': ['GET', 'PUT', 'POST'] }], 'users': [], - 'world': ['GET'] } } diff --git a/pillar/settings.py b/pillar/settings.py index 1c37ed20..a0a9abb8 100644 --- a/pillar/settings.py +++ b/pillar/settings.py @@ -392,7 +392,7 @@ files_schema = { 'required': True, 'allowed': ["attract-web", "pillar", "cdnsun", "gcs"] }, - 'path': { + 'file_path': { 'type': 'string', #'required': True, 'unique': True,