286 lines
11 KiB
Python
286 lines
11 KiB
Python
import os
|
|
import json
|
|
from multiprocessing import Process
|
|
from bson import ObjectId
|
|
from flask import request
|
|
from flask import Blueprint
|
|
from flask import abort
|
|
from flask import jsonify
|
|
from flask import send_from_directory
|
|
from flask import url_for
|
|
from eve.methods.put import put_internal
|
|
from application import app
|
|
from application.utils.imaging import generate_local_thumbnails
|
|
from application.utils.imaging import get_video_data
|
|
from application.utils.imaging import ffmpeg_encode
|
|
from application.utils.storage import remote_storage_sync
|
|
from application.utils.storage import push_to_storage
|
|
from application.utils.cdn import hash_file_path
|
|
from application.utils.gcs import GoogleCloudStorageBucket
|
|
from application.utils.encoding import Encoder
|
|
|
|
file_storage = Blueprint('file_storage', __name__,
|
|
template_folder='templates',
|
|
static_folder='../../static/storage',)
|
|
|
|
|
|
@file_storage.route('/gcs/<bucket_name>/<subdir>/')
|
|
@file_storage.route('/gcs/<bucket_name>/<subdir>/<path:file_path>')
|
|
def browse_gcs(bucket_name, subdir, file_path=None):
|
|
"""Browse the content of a Google Cloud Storage bucket"""
|
|
|
|
# Initialize storage client
|
|
storage = GoogleCloudStorageBucket(bucket_name, subdir=subdir)
|
|
if file_path:
|
|
# If we provided a file_path, we try to fetch it
|
|
file_object = storage.Get(file_path)
|
|
if file_object:
|
|
# If it exists, return file properties in a dictionary
|
|
return jsonify(file_object)
|
|
else:
|
|
listing = storage.List(file_path)
|
|
return jsonify(listing)
|
|
# We always return an empty listing even if the directory does not
|
|
# exist. This can be changed later.
|
|
# return abort(404)
|
|
|
|
else:
|
|
listing = storage.List('')
|
|
return jsonify(listing)
|
|
|
|
|
|
#@file_storage.route('/build_thumbnails/<path:file_path>')
|
|
def build_thumbnails(file_path=None, file_id=None):
|
|
"""Given a file path or file ObjectId pointing to an image file, fetch it
|
|
and generate a set of predefined variations (using generate_local_thumbnails).
|
|
Return a list of dictionaries containing the various image properties and
|
|
variation properties.
|
|
"""
|
|
files_collection = app.data.driver.db['files']
|
|
if file_path:
|
|
# Search file with backend "pillar" and path=file_path
|
|
file_ = files_collection.find({"file_path": "{0}".format(file_path)})
|
|
file_ = file_[0]
|
|
|
|
if file_id:
|
|
file_ = files_collection.find_one({"_id": ObjectId(file_id)})
|
|
file_path = file_['name']
|
|
|
|
file_full_path = os.path.join(app.config['SHARED_DIR'], file_path[:2], file_path)
|
|
# Does the original file exist?
|
|
if not os.path.isfile(file_full_path):
|
|
return "", 404
|
|
else:
|
|
thumbnails = generate_local_thumbnails(file_full_path,
|
|
return_image_stats=True)
|
|
|
|
file_variations = []
|
|
for size, thumbnail in thumbnails.iteritems():
|
|
if thumbnail.get('exists'):
|
|
# If a thumbnail was already made, we just continue
|
|
continue
|
|
basename = os.path.basename(thumbnail['file_path'])
|
|
root, ext = os.path.splitext(basename)
|
|
file_variation = dict(
|
|
size=size,
|
|
format=ext[1:],
|
|
width=thumbnail['width'],
|
|
height=thumbnail['height'],
|
|
content_type=thumbnail['content_type'],
|
|
length=thumbnail['length'],
|
|
md5=thumbnail['md5'],
|
|
file_path=basename,
|
|
)
|
|
# XXX Inject is_public for size 't' (should be part of the upload),
|
|
# and currently we set it here and then on the fly during blob
|
|
# creation by simply parsing the extension of the filename. This is
|
|
# bad.
|
|
if size == 't':
|
|
file_variation['is_public'] = True
|
|
|
|
file_variations.append(file_variation)
|
|
|
|
return file_variations
|
|
|
|
|
|
@file_storage.route('/file', methods=['POST'])
|
|
@file_storage.route('/file/<path:file_name>')
|
|
def index(file_name=None):
|
|
#GET file
|
|
if file_name:
|
|
return send_from_directory(app.config['STORAGE_DIR'], file_name)
|
|
#POST file
|
|
file_name = request.form['name']
|
|
folder_name = file_name[:2]
|
|
file_folder_path = os.path.join(app.config['STORAGE_DIR'],
|
|
folder_name)
|
|
if not os.path.exists(file_folder_path):
|
|
os.mkdir(file_folder_path)
|
|
file_path = os.path.join(file_folder_path, file_name)
|
|
request.files['data'].save(file_path)
|
|
|
|
return "{}", 200
|
|
|
|
|
|
def process_file(src_file):
|
|
"""Process the file
|
|
"""
|
|
file_id = src_file['_id']
|
|
# Remove properties that do not belong in the collection
|
|
internal_fields = ['_id', '_etag', '_updated', '_created', '_status']
|
|
for field in internal_fields:
|
|
src_file.pop(field, None)
|
|
|
|
files_collection = app.data.driver.db['files']
|
|
file_abs_path = os.path.join(
|
|
app.config['SHARED_DIR'], src_file['name'][:2], src_file['name'])
|
|
|
|
src_file['length'] = os.stat(file_abs_path).st_size
|
|
content_type = src_file['content_type'].split('/')
|
|
src_file['format'] = content_type[1]
|
|
mime_type = content_type[0]
|
|
src_file['file_path'] = src_file['name']
|
|
|
|
if mime_type == 'image':
|
|
from PIL import Image
|
|
im = Image.open(file_abs_path)
|
|
res = im.size
|
|
src_file['width'] = res[0]
|
|
src_file['height'] = res[1]
|
|
# Generate previews
|
|
src_file['variations'] = build_thumbnails(file_id=file_id)
|
|
elif mime_type == 'video':
|
|
pass
|
|
# Generate variations
|
|
src_video_data = get_video_data(file_abs_path)
|
|
variations = {
|
|
'mp4': None,
|
|
'webm': None
|
|
}
|
|
if src_video_data['duration']:
|
|
src_file['duration'] = src_video_data['duration']
|
|
|
|
# Properly resize the video according to 720p and 1080p resolutions
|
|
if src_video_data['res_y'] < 1080:
|
|
res_y = 720
|
|
elif src_video_data['res_y'] >= 1080:
|
|
res_y = 1080
|
|
|
|
# Add variations property to the file
|
|
src_file['variations'] = []
|
|
# Create variations
|
|
for v in variations:
|
|
root, ext = os.path.splitext(src_file['name'])
|
|
filename = "{0}-{1}p.{2}".format(root, res_y, v)
|
|
video_duration = None
|
|
if src_video_data['duration']:
|
|
video_duration = src_video_data['duration']
|
|
|
|
file_variation = dict(
|
|
size="{0}p".format(res_y),
|
|
duration=video_duration,
|
|
format=v,
|
|
width=src_video_data['res_x'],
|
|
height=src_video_data['res_y'],
|
|
content_type="video/{0}".format(v),
|
|
length=0, # Available after encode
|
|
md5="", # Available after encode
|
|
file_path=filename,
|
|
)
|
|
# Append file variation
|
|
src_file['variations'].append(file_variation)
|
|
|
|
def encode(src_path, src_file, res_y):
|
|
# For every variation in the list call video_encode
|
|
# print "encoding {0}".format(variations)
|
|
if app.config['ENCODING_BACKEND'] == 'zencoder':
|
|
# Move the source file in place on the remote storage (which can
|
|
# be accessed from zencoder)
|
|
push_to_storage(str(src_file['project']), src_path)
|
|
j = Encoder.job_create(src_file)
|
|
try:
|
|
if j:
|
|
src_file['processing'] = dict(
|
|
status='pending',
|
|
job_id="{0}".format(j['process_id']),
|
|
backend=j['backend'])
|
|
# Add the processing status to the file object
|
|
r = put_internal('files',
|
|
src_file, **{'_id': ObjectId(file_id)})
|
|
pass
|
|
except KeyError:
|
|
pass
|
|
elif app.config['ENCODING_BACKEND'] == 'local':
|
|
for v in src_file['variations']:
|
|
path = ffmpeg_encode(src_path, v['format'], res_y)
|
|
# Update size data after encoding
|
|
v['length'] = os.stat(path).st_size
|
|
|
|
r = put_internal('files', src_file, **{'_id': ObjectId(file_id)})
|
|
# When all encodes are done, delete source file
|
|
sync_path = os.path.split(src_path)[0]
|
|
push_to_storage(str(src_file['project']), sync_path)
|
|
|
|
p = Process(target=encode, args=(file_abs_path, src_file, res_y))
|
|
p.start()
|
|
if mime_type != 'video':
|
|
# Sync the whole subdir
|
|
sync_path = os.path.split(file_abs_path)[0]
|
|
# push_to_storage(str(src_file['project']), sync_path)
|
|
p = Process(target=push_to_storage, args=(
|
|
str(src_file['project']), sync_path))
|
|
p.start()
|
|
|
|
# Update the original file with additional info, e.g. image resolution
|
|
r = put_internal('files', src_file, **{'_id': ObjectId(file_id)})
|
|
|
|
|
|
def delete_file(file_item):
|
|
def process_file_delete(file_item):
|
|
"""Given a file item, delete the actual file from the storage backend.
|
|
This function can be probably made self-calling."""
|
|
if file_item['backend'] == 'gcs':
|
|
storage = GoogleCloudStorageBucket(str(file_item['project']))
|
|
storage.Delete(file_item['file_path'])
|
|
# Delete any file variation found in the file_item document
|
|
if 'variations' in file_item:
|
|
for v in file_item['variations']:
|
|
storage.Delete(v['file_path'])
|
|
return True
|
|
elif file_item['backend'] == 'pillar':
|
|
pass
|
|
elif file_item['backend'] == 'cdnsun':
|
|
pass
|
|
else:
|
|
pass
|
|
files_collection = app.data.driver.db['files']
|
|
# Collect children (variations) of the original file
|
|
children = files_collection.find({'parent': file_item['_id']})
|
|
for child in children:
|
|
process_file_delete(child)
|
|
# Finally remove the original file
|
|
process_file_delete(file_item)
|
|
|
|
|
|
def generate_link(backend, file_path, project_id=None, is_public=False):
|
|
"""Hook to check the backend of a file resource, to build an appropriate link
|
|
that can be used by the client to retrieve the actual file.
|
|
"""
|
|
if backend == 'gcs':
|
|
storage = GoogleCloudStorageBucket(project_id)
|
|
blob = storage.Get(file_path)
|
|
if blob and not is_public:
|
|
link = blob['signed_url']
|
|
elif blob and is_public:
|
|
link = blob['public_url']
|
|
else:
|
|
link = None
|
|
elif backend == 'pillar':
|
|
link = url_for('file_storage.index', file_name=file_path, _external=True,
|
|
_scheme=app.config['SCHEME'])
|
|
elif backend == 'cdnsun':
|
|
link = hash_file_path(file_path, None)
|
|
else:
|
|
link = None
|
|
return link
|