Initial commit for thumbnailing system

docker/Dockerfile

@@ -6,7 +6,10 @@ python \
 python-dev \
 python-pip \
 git \
-nano
+nano \
+zlib1g-dev \
+libjpeg-dev \
+
 
 RUN mkdir /data
 RUN mkdir /data/www

pillar/application/__init__.py

@@ -2,6 +2,7 @@ import os
 import json
 
 from eve import Eve
+from pymongo import MongoClient
 
 # import random
 # import string
@@ -226,6 +227,11 @@ app = Eve(validator=ValidateCustomFields, auth=CustomTokenAuth)
 
 import config
 app.config.from_object(config.Deployment)
+app.config['MONGO_HOST'] = os.environ.get('MONGO_HOST', 'localhost')
+
+client = MongoClient(app.config['MONGO_HOST'], 27017)
+db = client.eve
+
 
 def global_validation():
     setattr(g, 'token_data', validate_token())
@@ -292,7 +298,7 @@ def post_GET_user(request, payload):
 
 app.on_post_GET_users += post_GET_user
 
-from utils import hash_file_path
+from utils.cdn import hash_file_path
 # Hook to check the backend of a file resource, to build an appropriate link
 # that can be used by the client to retrieve the actual file.
 def generate_link(backend, path):

pillar/application/file_server.py

@@ -1,17 +1,15 @@
 import os
 import hashlib
-
+from datetime import datetime
+from PIL import Image
+from bson import ObjectId
 from flask import Blueprint
 from flask import request
-
 from application import app
+from application import db
 from application import post_item
+from application.utils.imaging import generate_local_thumbnails
 
-from datetime import datetime
-
-from PIL import Image
-
-from bson import ObjectId
 
 RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
 
@@ -29,131 +27,47 @@ def hashfile(afile, hasher, blocksize=65536):
     return hasher.hexdigest()
 
 
-@file_server.route('/build_previews/<file_name>')
+@file_server.route('/build_thumbnails/<path:file_path>')
-def build_previews(file_name=None):
+def build_thumbnails(file_path):
-    from pymongo import MongoClient
+    # Search file with backend "pillar" and path=file_path
-
+    file_ = db.files.find({"path": "{0}".format(file_path)})
-    # Get File
-    client = MongoClient()
-    db = client.eve
-    file_ = db.files.find({"path": "{0}".format(file_name)})
     file_ = file_[0]
     user = file_['user']
 
-    folder_name = file_name[:2]
+    file_full_path = os.path.join(app.config['FILE_STORAGE'],file_path)
-    file_folder_path = os.path.join(app.config['FILE_STORAGE'],
+    # Does the original file exist?
-                                    folder_name)
+    if not os.path.isfile(file_full_path):
-    # The original file exists?
-    file_path = os.path.join(file_folder_path, file_name)
-    if not os.path.isfile(file_path):
         return "", 404
+    else:
+        thumbnails = generate_local_thumbnails(file_full_path,
+            return_image_stats=True)
 
-    sizes = ["xs", "s", "m", "l", "xl"]
+    for size, thumbnail in thumbnails.iteritems():
-    size_dict = {
+        if thumbnail.get('exists'):
-        "xs": (32, 32),
+            # If a thumbnail was already made, we just continue
-        "s": (64, 64),
+            continue
-        "m": (128, 128),
+        basename = os.path.basename(thumbnail['path'])
-        "l": (640, 480),
+        root, ext = os.path.splitext(basename)
-        "xl": (1024, 768)
+        path = os.path.join(basename[:2], basename)
-        }
+        file_object = dict(
-
+            name=root,
-    # Generate
+            description="Preview of file {0}".format(file_['name']),
-    preview_list = []
+            user=user,
-    for size in sizes:
+            parent=file_['_id'],
-        resized_file_name = "{0}_{1}".format(size, file_name)
+            size=size,
-        resized_file_path = os.path.join(
+            format=ext[1:],
-            app.config['FILE_STORAGE'],
+            width=thumbnail['width'],
-            resized_file_name)
+            height=thumbnail['height'],
-
+            content_type=thumbnail['content_type'],
-        # Create thumbnail
+            length=thumbnail['length'],
-        #if not os.path.isfile(resized_file_path):
+            md5=thumbnail['md5'],
-        try:
+            filename=basename,
-            im = Image.open(file_path)
+            backend='pillar',
-        except IOError:
+            path=path)
-            return "", 500
+        # Commit to database
-        im.thumbnail(size_dict[size])
+        r = post_item('files', file_object)
-        width = im.size[0]
+        if r[0]['_status'] == 'ERR':
-        height = im.size[1]
+            return "", r[3] # The error code from the request
-        format = im.format.lower()
-        try:
-            im.save(resized_file_path)
-        except IOError:
-            return "", 500
-
-        # file_static_path = os.path.join("", folder_name, size, file_name)
-        picture_file_file = open(resized_file_path, 'rb')
-        hash_ = hashfile(picture_file_file, hashlib.md5())
-        name = "{0}{1}".format(hash_,
-                                os.path.splitext(file_name)[1])
-        picture_file_file.close()
-        description = "Thumbnail {0} for file {1}".format(
-            size, file_name)
-
-        prop = {}
-        prop['name'] = resized_file_name
-        prop['description'] = description
-        prop['user'] = user
-        # Preview properties:
-        prop['is_preview'] = True
-        prop['size'] = size
-        prop['format'] = format
-        prop['width'] = width
-        prop['height'] = height
-        # TODO set proper contentType and length
-        prop['contentType'] = 'image/png'
-        prop['length'] = 0
-        prop['uploadDate'] = datetime.strftime(
-            datetime.now(), RFC1123_DATE_FORMAT)
-        prop['md5'] = hash_
-        prop['filename'] = resized_file_name
-        prop['backend'] = 'attract'
-        prop['path'] = name
-
-        entry = post_item ('files', prop)
-        if entry[0]['_status'] == 'ERR':
-            entry = db.files.find({"path": name})
-
-        entry = entry[0]
-        prop['_id'] = entry['_id']
-
-        new_folder_name = name[:2]
-        new_folder_path = os.path.join(
-            app.config['FILE_STORAGE'],
-            new_folder_name)
-        new_file_path = os.path.join(
-            new_folder_path,
-            name)
-
-        if not os.path.exists(new_folder_path):
-            os.makedirs(new_folder_path)
-
-        # Clean up temporary file
-        os.rename(
-            resized_file_path,
-            new_file_path)
-
-        preview_list.append(str(prop['_id']))
-        #print (new_file_path)
-
-    # Add previews to file
-    previews = []
-    try:
-        previews = file_['previews']
-    except KeyError:
-        pass
-
-    preview_list = preview_list + previews
-
-    #print (previews)
-    #print (preview_list)
-    #print (file_['_id'])
-
-    file_ = db.files.update(
-        {"_id": ObjectId(file_['_id'])},
-        {"$set": {"previews": preview_list}}
-    )
-
-    #print (file_)
 
     return "", 200
 

pillar/application/utils/imaging.py

@@ -0,0 +1,112 @@
+import os
+from PIL import Image
+from application import app
+
+
+def generate_local_thumbnails(src, return_image_stats=False):
+    """Given a source image, use Pillow to generate thumbnails according to the
+    application settings.
+
+    args:
+    src: the path of the image to be thumbnailed
+    return_image_stats: if True, return a dict object which contains length,
+    resolution, format and path of the thumbnailed image
+    """
+
+    thumbnail_settings = app.config['UPLOADS_LOCAL_STORAGE_THUMBNAILS']
+    thumbnails = {}
+    for size, settings in thumbnail_settings.iteritems():
+        root, ext = os.path.splitext(src)
+        dst = "{0}-{1}{2}".format(root, size, '.jpg')
+        if os.path.isfile(dst):
+            # If the thumbnail already exists we require stats about it
+            if return_image_stats:
+                thumbnails[size] = dict(exists=True)
+            continue
+        if settings['crop']:
+            resize_and_crop(src, dst, settings['size'])
+        else:
+            im = Image.open(src)
+            im.thumbnail(settings['size'])
+            im.save(dst, "JPEG")
+
+        if return_image_stats:
+            # Get file size
+            st = os.stat(dst)
+            length = st.st_size
+            # Get resolution
+            im = Image.open(dst)
+            width = im.size[0]
+            height = im.size[1]
+            format = im.format.lower()
+            # Get format
+            thumbnails[size] = dict(
+                path=dst, # Full path, to be processed before storage
+                length=length,
+                width=width,
+                height=height,
+                md5='--',
+                content_type='image/' + format,
+                )
+
+    if return_image_stats:
+        return thumbnails
+
+
+def resize_and_crop(img_path, modified_path, size, crop_type='middle'):
+    """
+    Resize and crop an image to fit the specified size. Thanks to:
+    https://gist.github.com/sigilioso/2957026
+
+    args:
+    img_path: path for the image to resize.
+    modified_path: path to store the modified image.
+    size: `(width, height)` tuple.
+    crop_type: can be 'top', 'middle' or 'bottom', depending on this
+    value, the image will cropped getting the 'top/left', 'middle' or
+    'bottom/right' of the image to fit the size.
+    raises:
+    Exception: if can not open the file in img_path of there is problems
+    to save the image.
+    ValueError: if an invalid `crop_type` is provided.
+
+    """
+    # If height is higher we resize vertically, if not we resize horizontally
+    img = Image.open(img_path)
+    # Get current and desired ratio for the images
+    img_ratio = img.size[0] / float(img.size[1])
+    ratio = size[0] / float(size[1])
+    #The image is scaled/cropped vertically or horizontally depending on the ratio
+    if ratio > img_ratio:
+        img = img.resize((size[0], int(round(size[0] * img.size[1] / img.size[0]))),
+            Image.ANTIALIAS)
+        # Crop in the top, middle or bottom
+        if crop_type == 'top':
+            box = (0, 0, img.size[0], size[1])
+        elif crop_type == 'middle':
+            box = (0, int(round((img.size[1] - size[1]) / 2)), img.size[0],
+                int(round((img.size[1] + size[1]) / 2)))
+        elif crop_type == 'bottom':
+            box = (0, img.size[1] - size[1], img.size[0], img.size[1])
+        else :
+            raise ValueError('ERROR: invalid value for crop_type')
+        img = img.crop(box)
+    elif ratio < img_ratio:
+        img = img.resize((int(round(size[1] * img.size[0] / img.size[1])), size[1]),
+            Image.ANTIALIAS)
+        # Crop in the top, middle or bottom
+        if crop_type == 'top':
+            box = (0, 0, size[0], img.size[1])
+        elif crop_type == 'middle':
+            box = (int(round((img.size[0] - size[0]) / 2)), 0,
+                int(round((img.size[0] + size[0]) / 2)), img.size[1])
+        elif crop_type == 'bottom':
+            box = (img.size[0] - size[0], 0, img.size[0], img.size[1])
+        else :
+            raise ValueError('ERROR: invalid value for crop_type')
+        img = img.crop(box)
+    else :
+        img = img.resize((size[0], size[1]),
+            Image.ANTIALIAS)
+    # If the scale is the same, we do not need to crop
+    img.save(modified_path, "JPEG")

pillar/manage.py

@@ -1,5 +1,6 @@
 import os
 from application import app
+from application import db
 from application import post_item
 from flask.ext.script import Manager
 
@@ -673,7 +674,7 @@ def add_file_video():
         'name': 'Video test',
         'description': 'Video test description',
         # 'parent': 'objectid',
-        'contentType': 'video/mp4',
+        'content_type': 'video/mp4',
         # Duration in seconds, only if it's a video
         'duration': 50,
         'size': '720p',
@@ -707,7 +708,7 @@ def add_node_asset(file_id):
     file_object = db.files.find_one({"_id": ObjectId(file_id)})
     node_type = db.node_types.find_one({"name": "asset"})
 
-    print file_object['contentType'].split('/')[0]
+    print file_object['content_type'].split('/')[0]
 
     node = {
         'name': file_object['name'],
@@ -718,7 +719,7 @@ def add_node_asset(file_id):
         'node_type': node_type['_id'],
         'properties': {
             'status': 'published',
-            'contentType': file_object['contentType'].split('/')[0],
+            'content_type': file_object['content_type'].split('/')[0],
             'file': file_id
             }
         }
@@ -900,6 +901,19 @@ def import_data(path):
         json.dump(d, outfile, default=json_util.default)
     return
 
+@manager.command
+def make_thumbnails():
+    from application.file_server import build_thumbnails
+    files = db.files.find()
+    for f in files:
+        if f['content_type'].split('/')[0] == 'image':
+
+            if '-' in f['path']:
+                print "Skipping {0}".format(f['path'])
+            else:
+                print "Building {0}".format(f['path'])
+                t = build_thumbnails(f['path'])
+                print t
 
 
 if __name__ == '__main__':