archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Tweaks to push_to_storage

Browse Source 
Keep file removal, but do not remove file when encoding video until
everything is encoded. Also limit the response from GCS Post. Using
blob_to_dict leads to threading issues (due to signed url generation).
Good fix if you want to get involved in Pillar development!
This commit is contained in:
Francesco Siddi 2015-11-05 12:36:30 +01:00
parent 1819886af5
commit 4e5d076f49
2 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pillar/application/modules/file_storage/__init__.py
View File

@ -206,19 +206,21 @@ def process_file(src_file):
variation) variation)
# rsync the file file (this is async) # rsync the file file (this is async)
#remote_storage_sync(path) # remote_storage_sync(path)
# When all encodes are done, delete source file # push_to_storage(str(src_file['project']), path)
# When all encodes are done, delete source file
sync_path = os.path.split(file_abs_path)[0]
push_to_storage(str(src_file['project']), sync_path)
p = Process(target=encode, args=(file_abs_path, variations, res_y)) p = Process(target=encode, args=(file_abs_path, variations, res_y))
p.start() p.start()
if mime_type != 'video': if mime_type != 'video':
# Sync the whole subdir # Sync the whole subdir
sync_path = os.path.split(file_abs_path)[0] sync_path = os.path.split(file_abs_path)[0]
push_to_storage(str(src_file['project']), sync_path)
else: else:
sync_path = file_abs_path sync_path = file_abs_path
#remote_storage_sync(sync_path) #remote_storage_sync(sync_path)
push_to_storage(str(src_file['project']), sync_path)
# Update the original file with additional info, e.g. image resolution # Update the original file with additional info, e.g. image resolution
file_asset = files_collection.find_and_modify( file_asset = files_collection.find_and_modify(

5
pillar/application/utils/gcs.py
View File

@ -120,5 +120,8 @@ class GoogleCloudStorageBucket(object):
""" """
path = path if path else os.path.join('_', os.path.basename(full_path)) path = path if path else os.path.join('_', os.path.basename(full_path))
blob = self.bucket.blob(path) blob = self.bucket.blob(path)
if blob.exists():
return None
blob.upload_from_filename(full_path) blob.upload_from_filename(full_path)
return self.blob_to_dict(blob) return blob
# return self.blob_to_dict(blob) # Has issues with threading