From 00f24bb57e640b99c53ff8203415576a02ee7205 Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Fri, 11 Sep 2015 15:04:25 +0200 Subject: [PATCH] Initial commit for thumbnailing system --- docker/Dockerfile | 5 +- pillar/application/__init__.py | 8 +- pillar/application/file_server.py | 168 +++++------------- pillar/application/utils/__init__.py | 0 pillar/application/{utils.py => utils/cdn.py} | 0 pillar/application/utils/imaging.py | 112 ++++++++++++ pillar/manage.py | 20 ++- 7 files changed, 181 insertions(+), 132 deletions(-) create mode 100644 pillar/application/utils/__init__.py rename pillar/application/{utils.py => utils/cdn.py} (100%) create mode 100644 pillar/application/utils/imaging.py diff --git a/docker/Dockerfile b/docker/Dockerfile index c68218aa..546e2596 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,10 @@ python \ python-dev \ python-pip \ git \ -nano +nano \ +zlib1g-dev \ +libjpeg-dev \ + RUN mkdir /data RUN mkdir /data/www diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index a193f3ff..36aff667 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -2,6 +2,7 @@ import os import json from eve import Eve +from pymongo import MongoClient # import random # import string @@ -226,6 +227,11 @@ app = Eve(validator=ValidateCustomFields, auth=CustomTokenAuth) import config app.config.from_object(config.Deployment) +app.config['MONGO_HOST'] = os.environ.get('MONGO_HOST', 'localhost') + +client = MongoClient(app.config['MONGO_HOST'], 27017) +db = client.eve + def global_validation(): setattr(g, 'token_data', validate_token()) @@ -292,7 +298,7 @@ def post_GET_user(request, payload): app.on_post_GET_users += post_GET_user -from utils import hash_file_path +from utils.cdn import hash_file_path # Hook to check the backend of a file resource, to build an appropriate link # that can be used by the client to retrieve the actual file. def generate_link(backend, path): diff --git a/pillar/application/file_server.py b/pillar/application/file_server.py index 55f388a9..53fc6696 100644 --- a/pillar/application/file_server.py +++ b/pillar/application/file_server.py @@ -1,17 +1,15 @@ import os import hashlib - +from datetime import datetime +from PIL import Image +from bson import ObjectId from flask import Blueprint from flask import request - from application import app +from application import db from application import post_item +from application.utils.imaging import generate_local_thumbnails -from datetime import datetime - -from PIL import Image - -from bson import ObjectId RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' @@ -29,131 +27,47 @@ def hashfile(afile, hasher, blocksize=65536): return hasher.hexdigest() -@file_server.route('/build_previews/') -def build_previews(file_name=None): - from pymongo import MongoClient - - # Get File - client = MongoClient() - db = client.eve - file_ = db.files.find({"path": "{0}".format(file_name)}) +@file_server.route('/build_thumbnails/') +def build_thumbnails(file_path): + # Search file with backend "pillar" and path=file_path + file_ = db.files.find({"path": "{0}".format(file_path)}) file_ = file_[0] user = file_['user'] - folder_name = file_name[:2] - file_folder_path = os.path.join(app.config['FILE_STORAGE'], - folder_name) - # The original file exists? - file_path = os.path.join(file_folder_path, file_name) - if not os.path.isfile(file_path): + file_full_path = os.path.join(app.config['FILE_STORAGE'],file_path) + # Does the original file exist? + if not os.path.isfile(file_full_path): return "", 404 + else: + thumbnails = generate_local_thumbnails(file_full_path, + return_image_stats=True) - sizes = ["xs", "s", "m", "l", "xl"] - size_dict = { - "xs": (32, 32), - "s": (64, 64), - "m": (128, 128), - "l": (640, 480), - "xl": (1024, 768) - } - - # Generate - preview_list = [] - for size in sizes: - resized_file_name = "{0}_{1}".format(size, file_name) - resized_file_path = os.path.join( - app.config['FILE_STORAGE'], - resized_file_name) - - # Create thumbnail - #if not os.path.isfile(resized_file_path): - try: - im = Image.open(file_path) - except IOError: - return "", 500 - im.thumbnail(size_dict[size]) - width = im.size[0] - height = im.size[1] - format = im.format.lower() - try: - im.save(resized_file_path) - except IOError: - return "", 500 - - # file_static_path = os.path.join("", folder_name, size, file_name) - picture_file_file = open(resized_file_path, 'rb') - hash_ = hashfile(picture_file_file, hashlib.md5()) - name = "{0}{1}".format(hash_, - os.path.splitext(file_name)[1]) - picture_file_file.close() - description = "Thumbnail {0} for file {1}".format( - size, file_name) - - prop = {} - prop['name'] = resized_file_name - prop['description'] = description - prop['user'] = user - # Preview properties: - prop['is_preview'] = True - prop['size'] = size - prop['format'] = format - prop['width'] = width - prop['height'] = height - # TODO set proper contentType and length - prop['contentType'] = 'image/png' - prop['length'] = 0 - prop['uploadDate'] = datetime.strftime( - datetime.now(), RFC1123_DATE_FORMAT) - prop['md5'] = hash_ - prop['filename'] = resized_file_name - prop['backend'] = 'attract' - prop['path'] = name - - entry = post_item ('files', prop) - if entry[0]['_status'] == 'ERR': - entry = db.files.find({"path": name}) - - entry = entry[0] - prop['_id'] = entry['_id'] - - new_folder_name = name[:2] - new_folder_path = os.path.join( - app.config['FILE_STORAGE'], - new_folder_name) - new_file_path = os.path.join( - new_folder_path, - name) - - if not os.path.exists(new_folder_path): - os.makedirs(new_folder_path) - - # Clean up temporary file - os.rename( - resized_file_path, - new_file_path) - - preview_list.append(str(prop['_id'])) - #print (new_file_path) - - # Add previews to file - previews = [] - try: - previews = file_['previews'] - except KeyError: - pass - - preview_list = preview_list + previews - - #print (previews) - #print (preview_list) - #print (file_['_id']) - - file_ = db.files.update( - {"_id": ObjectId(file_['_id'])}, - {"$set": {"previews": preview_list}} - ) - - #print (file_) + for size, thumbnail in thumbnails.iteritems(): + if thumbnail.get('exists'): + # If a thumbnail was already made, we just continue + continue + basename = os.path.basename(thumbnail['path']) + root, ext = os.path.splitext(basename) + path = os.path.join(basename[:2], basename) + file_object = dict( + name=root, + description="Preview of file {0}".format(file_['name']), + user=user, + parent=file_['_id'], + size=size, + format=ext[1:], + width=thumbnail['width'], + height=thumbnail['height'], + content_type=thumbnail['content_type'], + length=thumbnail['length'], + md5=thumbnail['md5'], + filename=basename, + backend='pillar', + path=path) + # Commit to database + r = post_item('files', file_object) + if r[0]['_status'] == 'ERR': + return "", r[3] # The error code from the request return "", 200 diff --git a/pillar/application/utils/__init__.py b/pillar/application/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pillar/application/utils.py b/pillar/application/utils/cdn.py similarity index 100% rename from pillar/application/utils.py rename to pillar/application/utils/cdn.py diff --git a/pillar/application/utils/imaging.py b/pillar/application/utils/imaging.py new file mode 100644 index 00000000..495fbea7 --- /dev/null +++ b/pillar/application/utils/imaging.py @@ -0,0 +1,112 @@ +import os +from PIL import Image +from application import app + + +def generate_local_thumbnails(src, return_image_stats=False): + """Given a source image, use Pillow to generate thumbnails according to the + application settings. + + args: + src: the path of the image to be thumbnailed + return_image_stats: if True, return a dict object which contains length, + resolution, format and path of the thumbnailed image + """ + + thumbnail_settings = app.config['UPLOADS_LOCAL_STORAGE_THUMBNAILS'] + thumbnails = {} + for size, settings in thumbnail_settings.iteritems(): + root, ext = os.path.splitext(src) + dst = "{0}-{1}{2}".format(root, size, '.jpg') + if os.path.isfile(dst): + # If the thumbnail already exists we require stats about it + if return_image_stats: + thumbnails[size] = dict(exists=True) + continue + if settings['crop']: + resize_and_crop(src, dst, settings['size']) + else: + im = Image.open(src) + im.thumbnail(settings['size']) + im.save(dst, "JPEG") + + if return_image_stats: + # Get file size + st = os.stat(dst) + length = st.st_size + # Get resolution + im = Image.open(dst) + width = im.size[0] + height = im.size[1] + format = im.format.lower() + # Get format + thumbnails[size] = dict( + path=dst, # Full path, to be processed before storage + length=length, + width=width, + height=height, + md5='--', + content_type='image/' + format, + ) + + if return_image_stats: + return thumbnails + + +def resize_and_crop(img_path, modified_path, size, crop_type='middle'): + """ + Resize and crop an image to fit the specified size. Thanks to: + https://gist.github.com/sigilioso/2957026 + + args: + img_path: path for the image to resize. + modified_path: path to store the modified image. + size: `(width, height)` tuple. + crop_type: can be 'top', 'middle' or 'bottom', depending on this + value, the image will cropped getting the 'top/left', 'middle' or + 'bottom/right' of the image to fit the size. + raises: + Exception: if can not open the file in img_path of there is problems + to save the image. + ValueError: if an invalid `crop_type` is provided. + + """ + # If height is higher we resize vertically, if not we resize horizontally + img = Image.open(img_path) + # Get current and desired ratio for the images + img_ratio = img.size[0] / float(img.size[1]) + ratio = size[0] / float(size[1]) + #The image is scaled/cropped vertically or horizontally depending on the ratio + if ratio > img_ratio: + img = img.resize((size[0], int(round(size[0] * img.size[1] / img.size[0]))), + Image.ANTIALIAS) + # Crop in the top, middle or bottom + if crop_type == 'top': + box = (0, 0, img.size[0], size[1]) + elif crop_type == 'middle': + box = (0, int(round((img.size[1] - size[1]) / 2)), img.size[0], + int(round((img.size[1] + size[1]) / 2))) + elif crop_type == 'bottom': + box = (0, img.size[1] - size[1], img.size[0], img.size[1]) + else : + raise ValueError('ERROR: invalid value for crop_type') + img = img.crop(box) + elif ratio < img_ratio: + img = img.resize((int(round(size[1] * img.size[0] / img.size[1])), size[1]), + Image.ANTIALIAS) + # Crop in the top, middle or bottom + if crop_type == 'top': + box = (0, 0, size[0], img.size[1]) + elif crop_type == 'middle': + box = (int(round((img.size[0] - size[0]) / 2)), 0, + int(round((img.size[0] + size[0]) / 2)), img.size[1]) + elif crop_type == 'bottom': + box = (img.size[0] - size[0], 0, img.size[0], img.size[1]) + else : + raise ValueError('ERROR: invalid value for crop_type') + img = img.crop(box) + else : + img = img.resize((size[0], size[1]), + Image.ANTIALIAS) + # If the scale is the same, we do not need to crop + img.save(modified_path, "JPEG") diff --git a/pillar/manage.py b/pillar/manage.py index c6d28380..ed860116 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -1,5 +1,6 @@ import os from application import app +from application import db from application import post_item from flask.ext.script import Manager @@ -673,7 +674,7 @@ def add_file_video(): 'name': 'Video test', 'description': 'Video test description', # 'parent': 'objectid', - 'contentType': 'video/mp4', + 'content_type': 'video/mp4', # Duration in seconds, only if it's a video 'duration': 50, 'size': '720p', @@ -707,7 +708,7 @@ def add_node_asset(file_id): file_object = db.files.find_one({"_id": ObjectId(file_id)}) node_type = db.node_types.find_one({"name": "asset"}) - print file_object['contentType'].split('/')[0] + print file_object['content_type'].split('/')[0] node = { 'name': file_object['name'], @@ -718,7 +719,7 @@ def add_node_asset(file_id): 'node_type': node_type['_id'], 'properties': { 'status': 'published', - 'contentType': file_object['contentType'].split('/')[0], + 'content_type': file_object['content_type'].split('/')[0], 'file': file_id } } @@ -900,6 +901,19 @@ def import_data(path): json.dump(d, outfile, default=json_util.default) return +@manager.command +def make_thumbnails(): + from application.file_server import build_thumbnails + files = db.files.find() + for f in files: + if f['content_type'].split('/')[0] == 'image': + + if '-' in f['path']: + print "Skipping {0}".format(f['path']) + else: + print "Building {0}".format(f['path']) + t = build_thumbnails(f['path']) + print t if __name__ == '__main__':