archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Initial commit for thumbnailing system

Browse Source
This commit is contained in:
Francesco Siddi 2015-09-11 15:04:25 +02:00
parent 4e92cfe6b6
commit 00f24bb57e
7 changed files with 181 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docker/Dockerfile
View File

@ -6,7 +6,10 @@ python \
python-dev \
python-pip \
git \
nano
nano \
zlib1g-dev \
libjpeg-dev \
RUN mkdir /data
RUN mkdir /data/www

8
pillar/application/__init__.py
View File

@ -2,6 +2,7 @@ import os
import json
from eve import Eve
from pymongo import MongoClient
# import random
# import string
@ -226,6 +227,11 @@ app = Eve(validator=ValidateCustomFields, auth=CustomTokenAuth)
import config
app.config.from_object(config.Deployment)
app.config['MONGO_HOST'] = os.environ.get('MONGO_HOST', 'localhost')
client = MongoClient(app.config['MONGO_HOST'], 27017)
db = client.eve
def global_validation():
setattr(g, 'token_data', validate_token())
@ -292,7 +298,7 @@ def post_GET_user(request, payload):
app.on_post_GET_users += post_GET_user
from utils import hash_file_path
from utils.cdn import hash_file_path
# Hook to check the backend of a file resource, to build an appropriate link
# that can be used by the client to retrieve the actual file.
def generate_link(backend, path):

168
pillar/application/file_server.py
View File

@ -1,17 +1,15 @@
import os
import hashlib
from datetime import datetime
from PIL import Image
from bson import ObjectId
from flask import Blueprint
from flask import request
from application import app
from application import db
from application import post_item
from application.utils.imaging import generate_local_thumbnails
from datetime import datetime
from PIL import Image
from bson import ObjectId
RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
@ -29,131 +27,47 @@ def hashfile(afile, hasher, blocksize=65536):
return hasher.hexdigest()
@file_server.route('/build_previews/<file_name>')
def build_previews(file_name=None):
from pymongo import MongoClient
# Get File
client = MongoClient()
db = client.eve
file_ = db.files.find({"path": "{0}".format(file_name)})
@file_server.route('/build_thumbnails/<path:file_path>')
def build_thumbnails(file_path):
# Search file with backend "pillar" and path=file_path
file_ = db.files.find({"path": "{0}".format(file_path)})
file_ = file_[0]
user = file_['user']
folder_name = file_name[:2]
file_folder_path = os.path.join(app.config['FILE_STORAGE'],
folder_name)
# The original file exists?
file_path = os.path.join(file_folder_path, file_name)
if not os.path.isfile(file_path):
file_full_path = os.path.join(app.config['FILE_STORAGE'],file_path)
# Does the original file exist?
if not os.path.isfile(file_full_path):
return "", 404
else:
thumbnails = generate_local_thumbnails(file_full_path,
return_image_stats=True)
sizes = ["xs", "s", "m", "l", "xl"]
size_dict = {
"xs": (32, 32),
"s": (64, 64),
"m": (128, 128),
"l": (640, 480),
"xl": (1024, 768)
}
# Generate
preview_list = []
for size in sizes:
resized_file_name = "{0}_{1}".format(size, file_name)
resized_file_path = os.path.join(
app.config['FILE_STORAGE'],
resized_file_name)
# Create thumbnail
#if not os.path.isfile(resized_file_path):
try:
im = Image.open(file_path)
except IOError:
return "", 500
im.thumbnail(size_dict[size])
width = im.size[0]
height = im.size[1]
format = im.format.lower()
try:
im.save(resized_file_path)
except IOError:
return "", 500
# file_static_path = os.path.join("", folder_name, size, file_name)
picture_file_file = open(resized_file_path, 'rb')
hash_ = hashfile(picture_file_file, hashlib.md5())
name = "{0}{1}".format(hash_,
os.path.splitext(file_name)[1])
picture_file_file.close()
description = "Thumbnail {0} for file {1}".format(
size, file_name)
prop = {}
prop['name'] = resized_file_name
prop['description'] = description
prop['user'] = user
# Preview properties:
prop['is_preview'] = True
prop['size'] = size
prop['format'] = format
prop['width'] = width
prop['height'] = height
# TODO set proper contentType and length
prop['contentType'] = 'image/png'
prop['length'] = 0
prop['uploadDate'] = datetime.strftime(
datetime.now(), RFC1123_DATE_FORMAT)
prop['md5'] = hash_
prop['filename'] = resized_file_name
prop['backend'] = 'attract'
prop['path'] = name
entry = post_item ('files', prop)
if entry[0]['_status'] == 'ERR':
entry = db.files.find({"path": name})
entry = entry[0]
prop['_id'] = entry['_id']
new_folder_name = name[:2]
new_folder_path = os.path.join(
app.config['FILE_STORAGE'],
new_folder_name)
new_file_path = os.path.join(
new_folder_path,
name)
if not os.path.exists(new_folder_path):
os.makedirs(new_folder_path)
# Clean up temporary file
os.rename(
resized_file_path,
new_file_path)
preview_list.append(str(prop['_id']))
#print (new_file_path)
# Add previews to file
previews = []
try:
previews = file_['previews']
except KeyError:
pass
preview_list = preview_list + previews
#print (previews)
#print (preview_list)
#print (file_['_id'])
file_ = db.files.update(
{"_id": ObjectId(file_['_id'])},
{"$set": {"previews": preview_list}}
)
#print (file_)
for size, thumbnail in thumbnails.iteritems():
if thumbnail.get('exists'):
# If a thumbnail was already made, we just continue
continue
basename = os.path.basename(thumbnail['path'])
root, ext = os.path.splitext(basename)
path = os.path.join(basename[:2], basename)
file_object = dict(
name=root,
description="Preview of file {0}".format(file_['name']),
user=user,
parent=file_['_id'],
size=size,
format=ext[1:],
width=thumbnail['width'],
height=thumbnail['height'],
content_type=thumbnail['content_type'],
length=thumbnail['length'],
md5=thumbnail['md5'],
filename=basename,
backend='pillar',
path=path)
# Commit to database
r = post_item('files', file_object)
if r[0]['_status'] == 'ERR':
return "", r[3] # The error code from the request
return "", 200

0
pillar/application/utils/__init__.py Normal file
View File

0
pillar/application/utils.py → pillar/application/utils/cdn.py
View File

112
pillar/application/utils/imaging.py Normal file
View File

@ -0,0 +1,112 @@
import os
from PIL import Image
from application import app
def generate_local_thumbnails(src, return_image_stats=False):
"""Given a source image, use Pillow to generate thumbnails according to the
application settings.
args:
src: the path of the image to be thumbnailed
return_image_stats: if True, return a dict object which contains length,
resolution, format and path of the thumbnailed image
"""
thumbnail_settings = app.config['UPLOADS_LOCAL_STORAGE_THUMBNAILS']
thumbnails = {}
for size, settings in thumbnail_settings.iteritems():
root, ext = os.path.splitext(src)
dst = "{0}-{1}{2}".format(root, size, '.jpg')
if os.path.isfile(dst):
# If the thumbnail already exists we require stats about it
if return_image_stats:
thumbnails[size] = dict(exists=True)
continue
if settings['crop']:
resize_and_crop(src, dst, settings['size'])
else:
im = Image.open(src)
im.thumbnail(settings['size'])
im.save(dst, "JPEG")
if return_image_stats:
# Get file size
st = os.stat(dst)
length = st.st_size
# Get resolution
im = Image.open(dst)
width = im.size[0]
height = im.size[1]
format = im.format.lower()
# Get format
thumbnails[size] = dict(
path=dst, # Full path, to be processed before storage
length=length,
width=width,
height=height,
md5='--',
content_type='image/' + format,
)
if return_image_stats:
return thumbnails
def resize_and_crop(img_path, modified_path, size, crop_type='middle'):
"""
Resize and crop an image to fit the specified size. Thanks to:
https://gist.github.com/sigilioso/2957026
args:
img_path: path for the image to resize.
modified_path: path to store the modified image.
size: `(width, height)` tuple.
crop_type: can be 'top', 'middle' or 'bottom', depending on this
value, the image will cropped getting the 'top/left', 'middle' or
'bottom/right' of the image to fit the size.
raises:
Exception: if can not open the file in img_path of there is problems
to save the image.
ValueError: if an invalid `crop_type` is provided.
"""
# If height is higher we resize vertically, if not we resize horizontally
img = Image.open(img_path)
# Get current and desired ratio for the images
img_ratio = img.size[0] / float(img.size[1])
ratio = size[0] / float(size[1])
#The image is scaled/cropped vertically or horizontally depending on the ratio
if ratio > img_ratio:
img = img.resize((size[0], int(round(size[0] * img.size[1] / img.size[0]))),
Image.ANTIALIAS)
# Crop in the top, middle or bottom
if crop_type == 'top':
box = (0, 0, img.size[0], size[1])
elif crop_type == 'middle':
box = (0, int(round((img.size[1] - size[1]) / 2)), img.size[0],
int(round((img.size[1] + size[1]) / 2)))
elif crop_type == 'bottom':
box = (0, img.size[1] - size[1], img.size[0], img.size[1])
else :
raise ValueError('ERROR: invalid value for crop_type')
img = img.crop(box)
elif ratio < img_ratio:
img = img.resize((int(round(size[1] * img.size[0] / img.size[1])), size[1]),
Image.ANTIALIAS)
# Crop in the top, middle or bottom
if crop_type == 'top':
box = (0, 0, size[0], img.size[1])
elif crop_type == 'middle':
box = (int(round((img.size[0] - size[0]) / 2)), 0,
int(round((img.size[0] + size[0]) / 2)), img.size[1])
elif crop_type == 'bottom':
box = (img.size[0] - size[0], 0, img.size[0], img.size[1])
else :
raise ValueError('ERROR: invalid value for crop_type')
img = img.crop(box)
else :
img = img.resize((size[0], size[1]),
Image.ANTIALIAS)
# If the scale is the same, we do not need to crop
img.save(modified_path, "JPEG")

20
pillar/manage.py
View File

@ -1,5 +1,6 @@
import os
from application import app
from application import db
from application import post_item
from flask.ext.script import Manager
@ -673,7 +674,7 @@ def add_file_video():
'name': 'Video test',
'description': 'Video test description',
# 'parent': 'objectid',
'contentType': 'video/mp4',
'content_type': 'video/mp4',
# Duration in seconds, only if it's a video
'duration': 50,
'size': '720p',
@ -707,7 +708,7 @@ def add_node_asset(file_id):
file_object = db.files.find_one({"_id": ObjectId(file_id)})
node_type = db.node_types.find_one({"name": "asset"})
print file_object['contentType'].split('/')[0]
print file_object['content_type'].split('/')[0]
node = {
'name': file_object['name'],
@ -718,7 +719,7 @@ def add_node_asset(file_id):
'node_type': node_type['_id'],
'properties': {
'status': 'published',
'contentType': file_object['contentType'].split('/')[0],
'content_type': file_object['content_type'].split('/')[0],
'file': file_id
}
}
@ -900,6 +901,19 @@ def import_data(path):
json.dump(d, outfile, default=json_util.default)
return
@manager.command
def make_thumbnails():
from application.file_server import build_thumbnails
files = db.files.find()
for f in files:
if f['content_type'].split('/')[0] == 'image':
if '-' in f['path']:
print "Skipping {0}".format(f['path'])
else:
print "Building {0}".format(f['path'])
t = build_thumbnails(f['path'])
print t
if __name__ == '__main__':