constants/base.py

@@ -100,3 +100,13 @@ ABUSE_TYPE = Choices(
     ('ABUSE_USER', ABUSE_TYPE_USER, "User"),
     ('ABUSE_RATING', ABUSE_TYPE_RATING, "Rating"),
 )
+
+# **N.B.**: thumbnail sizes are not intended to be changed on the fly:
+# thumbnails of existing images must exist in MEDIA_ROOT before
+# the code expecting the new dimensions can be deployed!
+THUMBNAIL_SIZE_DEFAULT = (1920, 1080)
+THUMBNAIL_SIZE_M = (1280, 720)
+THUMBNAIL_SIZE_S = (640, 360)
+THUMBNAIL_SIZES = [THUMBNAIL_SIZE_DEFAULT, THUMBNAIL_SIZE_M, THUMBNAIL_SIZE_S]
+THUMBNAIL_FORMAT = 'PNG'
+THUMBNAIL_QUALITY = 83

extensions/templates/extensions/components/card.html

@@ -1,13 +1,13 @@
 {% load common filters %}
-{% with latest=extension.latest_version %}
+{% with latest=extension.latest_version thumbnail_url=extension.previews.listed.first.thumbnail_url %}
 
 <div class="ext-card {% if blur %}is-background-blur{% endif %}">
   {% if blur %}
-  <div class="ext-card-thumbnail-blur" style="background-image: url({{ extension.previews.listed.first.source.url }});"></div>
+  <div class="ext-card-thumbnail-blur" style="background-image: url({{ thumbnail_url }});"></div>
   {% endif %}
 
   <a class="ext-card-thumbnail" href="{{ extension.get_absolute_url }}">
-    <div class="ext-card-thumbnail-img" style="background-image: url({{ extension.previews.listed.first.source.url }});" title="{{ extension.name }}"></div>
+    <div class="ext-card-thumbnail-img" style="background-image: url({{ thumbnail_url }});" title="{{ extension.name }}"></div>
   </a>
 
   <div class="ext-card-body">

extensions/templates/extensions/components/galleria.html

@@ -3,19 +3,17 @@
   {% if previews %}
   <div class="galleria-items{% if previews.count > 5 %} is-many{% endif %}{% if previews.count == 1 %} is-single{% endif %}" id="galleria-items">
     {% for preview in previews %}
-    <a
-      class="galleria-item js-galleria-item-preview galleria-item-type-{{ preview.content_type|slugify|slice:5 }}{% if forloop.first %} is-active{% endif %}"
-      href="{{ preview.source.url }}"
-      {% if 'video' in preview.content_type %}data-galleria-video-url="{{ preview.source.url }}"{% endif %}
-      data-galleria-content-type="{{ preview.content_type }}"
-      data-galleria-index="{{ forloop.counter }}">
+      {% with thumbnail_url=preview.thumbnail_url %}
+      <a
+        class="galleria-item js-galleria-item-preview galleria-item-type-{{ preview.content_type|slugify|slice:5 }}{% if forloop.first %} is-active{% endif %}"
+        href="{{ thumbnail_url }}"
+        {% if 'video' in preview.content_type %}data-galleria-video-url="{{ preview.source.url }}"{% endif %}
+        data-galleria-content-type="{{ preview.content_type }}"
+        data-galleria-index="{{ forloop.counter }}">
 
-        {% if 'video' in preview.content_type and preview.thumbnail %}
-        <img src="{{ preview.thumbnail.url }}" alt="{{ preview.extension_preview.first.caption }}">
-        {% else %}
-        <img src="{{ preview.source.url }}" alt="{{ preview.extension_preview.first.caption }}">
-        {% endif %}
-      </a>
+        <img src="{{ thumbnail_url }}" alt="{{ preview.extension_preview.first.caption }}">
+        </a>
+      {% endwith %}
     {% endfor %}
   </div>
   {% else %}

files/admin.py

@@ -1,10 +1,17 @@
+import logging
+
+from django.conf import settings
 from django.contrib import admin
+from django.template.loader import render_to_string
 import background_task.admin
 import background_task.models
 
 from .models import File, FileValidation
+from constants.base import THUMBNAIL_SIZE_DEFAULT
 import files.signals
 
+logger = logging.getLogger(__name__)
+
 
 def scan_selected_files(self, request, queryset):
     """Scan selected files."""
@@ -12,6 +19,12 @@ def scan_selected_files(self, request, queryset):
         files.signals.schedule_scan(instance)
 
 
+def make_thumbnails(self, request, queryset):
+    """Make thumbnails for selected files."""
+    for instance in queryset:
+        files.tasks.make_thumbnails.task_function(file_id=instance.pk)
+
+
 class FileValidationInlineAdmin(admin.StackedInline):
     model = FileValidation
     readonly_fields = ('date_created', 'date_modified', 'is_ok', 'results')
@@ -27,6 +40,24 @@ class FileValidationInlineAdmin(admin.StackedInline):
 
 @admin.register(File)
 class FileAdmin(admin.ModelAdmin):
+    class Media:
+        css = {'all': ('files/admin/file.css',)}
+
+    def thumbnails(self, obj):
+        try:
+            context = {
+                'MEDIA_URL': settings.MEDIA_URL,
+                'THUMBNAIL_SIZE_DEFAULT': THUMBNAIL_SIZE_DEFAULT,
+                'thumbnail': obj.thumbnail,
+                'thumbnails': obj.thumbnails,
+            }
+            return render_to_string('files/admin/thumbnails.html', context)
+        except Exception:
+            # Make sure any exception happening here is always logged
+            # (e.g. admin eats exception in ModelAdmin properties, making it hard to debug)
+            logger.exception('Failed to render thumbnails')
+            raise
+
     view_on_site = False
     save_on_top = True
 
@@ -48,6 +79,9 @@ class FileAdmin(admin.ModelAdmin):
         'date_approved',
         'date_status_changed',
         'size_bytes',
+        'thumbnail',
+        'thumbnails',
+        'type',
         'user',
         'original_hash',
         'original_name',
@@ -67,9 +101,8 @@ class FileAdmin(admin.ModelAdmin):
             {
                 'fields': (
                     'id',
-                    ('source', 'thumbnail'),
-                    ('original_name', 'content_type'),
-                    'type',
+                    ('source', 'thumbnail', 'thumbnails'),
+                    ('type', 'content_type', 'original_name'),
                     'status',
                 )
             },
@@ -99,7 +132,7 @@ class FileAdmin(admin.ModelAdmin):
     )
 
     inlines = [FileValidationInlineAdmin]
-    actions = [scan_selected_files]
+    actions = [scan_selected_files, make_thumbnails]
 
     def is_ok(self, obj):
         return obj.validation.is_ok if hasattr(obj, 'validation') else None

files/models.py

@@ -6,10 +6,12 @@ from django.contrib.auth import get_user_model
 from django.db import models
 
 from common.model_mixins import CreatedModifiedMixin, TrackChangesMixin
-from files.utils import get_sha256, guess_mimetype_from_ext
+from files.utils import get_sha256, guess_mimetype_from_ext, get_base_path
 from constants.base import (
     FILE_STATUS_CHOICES,
     FILE_TYPE_CHOICES,
+    THUMBNAIL_FORMAT,
+    THUMBNAIL_SIZES,
 )
 import utils
 
@@ -43,13 +45,13 @@ def file_upload_to(instance, filename):
 def thumbnail_upload_to(instance, filename):
     prefix = 'thumbnails/'
     _hash = instance.hash.split(':')[-1]
-    extension = Path(filename).suffix
+    extension = f'.{THUMBNAIL_FORMAT.lower()}'
     path = Path(prefix, _hash[:2], _hash).with_suffix(extension)
     return path
 
 
 class File(CreatedModifiedMixin, TrackChangesMixin, models.Model):
-    track_changes_to_fields = {'status', 'size_bytes', 'hash'}
+    track_changes_to_fields = {'status', 'size_bytes', 'hash', 'thumbnail'}
 
     TYPES = FILE_TYPE_CHOICES
     STATUSES = FILE_STATUS_CHOICES
@@ -63,7 +65,8 @@ class File(CreatedModifiedMixin, TrackChangesMixin, models.Model):
         null=True,
         blank=True,
         max_length=256,
-        help_text='Image thumbnail in case file is a video',
+        help_text='Image thumbnail',
+        editable=False,
     )
     content_type = models.CharField(max_length=256, null=True, blank=True)
     type = models.PositiveSmallIntegerField(
@@ -203,6 +206,22 @@ class File(CreatedModifiedMixin, TrackChangesMixin, models.Model):
     def get_submit_url(self) -> str:
         return self.extension.get_draft_url()
 
+    @property
+    def thumbnails(self) -> dict:
+        """Return a dictionary with relative URLs of a set of predefined thumbnails."""
+        base_path = get_base_path(thumbnail_upload_to(self, ''))
+        extension = f'.{THUMBNAIL_FORMAT.lower()}'
+        return {size: f'{base_path}_{size[0]}x{size[1]}{extension}' for size in THUMBNAIL_SIZES}
+
+    @property
+    def thumbnail_url(self) -> str:
+        """Log absence of the thumbnail file instead of exploding somewhere in the templates."""
+        try:
+            return self.thumbnail.url
+        except ValueError:
+            log.exception(f'File pk={self.pk} is missing thumbnail(s)')
+        return ''
+
 
 class FileValidation(CreatedModifiedMixin, TrackChangesMixin, models.Model):
     track_changes_to_fields = {'is_ok', 'results'}

files/signals.py

@@ -1,10 +1,11 @@
 import logging
 
-from django.db.models.signals import pre_save, post_save, pre_delete
+from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
 from django.dispatch import receiver
 
 import files.models
 import files.tasks
+import files.utils
 
 logger = logging.getLogger(__name__)
 
@@ -35,7 +36,35 @@ def _scan_new_file(
     schedule_scan(instance)
 
 
+def schedule_thumbnails(file: files.models.File) -> None:
+    """Schedule thumbnail generation for a given file."""
+    args = {'pk': file.pk, 'type': file.get_type_display()}
+    logger.info('Scheduling thumbnail generation for file pk=%s type=%s', args)
+    verbose_name = f'make thumbnails for "{file.source.name}"'
+    files.tasks.make_thumbnails(file_id=file.pk, creator=file, verbose_name=verbose_name)
+
+
+@receiver(post_save, sender=files.models.FileValidation)
+def _schedule_thumbnails(
+    sender: object, instance: files.models.FileValidation, created: bool, **kwargs: object
+) -> None:
+    if not created:
+        return
+
+    file = instance.file
+    if instance.is_ok and (file.is_image or file.is_video):
+        # Generate thumbnails if initial scan found no issues
+        schedule_thumbnails(file)
+
+
 @receiver(pre_delete, sender=files.models.File)
 @receiver(pre_delete, sender=files.models.FileValidation)
 def _log_deletion(sender: object, instance: files.models.File, **kwargs: object) -> None:
     instance.record_deletion()
+
+
+@receiver(post_delete, sender=files.models.File)
+def delete_orphaned_files(sender: object, instance: files.models.File, **kwargs: object) -> None:
+    """Delete source and thumbnail files from storage when File record is deleted."""
+    files.utils.delete_source_file(instance.source.name)
+    files.utils.delete_thumbnail_file(instance.thumbnail.name)

files/static/files/admin/file.css

@@ -0,0 +1,14 @@
+.file-thumbnail {
+    display: inline-block;
+    border: grey solid 1px;
+    margin-left: 0.5rem;
+}
+.file-thumbnail-size {
+    position: absolute;
+    background: rgba(255, 255, 255, 0.5);
+    padding-right: 0.5rem;
+    padding-left: 0.5rem;
+}
+.file-thumbnails .icon-alert {
+    color: red;
+}

files/tasks.py

@@ -5,6 +5,7 @@ from background_task import background
 from background_task.tasks import TaskSchedule
 from django.conf import settings
 
+from constants.base import THUMBNAIL_FORMAT, THUMBNAIL_SIZE_DEFAULT, THUMBNAIL_QUALITY
 import files.models
 import files.utils
 
@@ -27,3 +28,33 @@ def clamdscan(file_id: int):
         file_validation.results = scan_result
         file_validation.is_ok = is_ok
         file_validation.save(update_fields={'results', 'is_ok', 'date_modified'})
+
+
+@background(schedule={'action': TaskSchedule.RESCHEDULE_EXISTING})
+def make_thumbnails(file_id: int) -> None:
+    """Generate thumbnails for a given file."""
+    file = files.models.File.objects.get(pk=file_id)
+    abs_path = os.path.join(settings.MEDIA_ROOT, file.source.path)
+
+    # TODO: For an image, source of the thumbnail is file.source itself
+    if file.is_image:
+        image_field = file.thumbnail
+        if not image_field:
+            # Use the source image if max-size thumbnail is not yet available
+            image_field.name = file.source.name
+        # base_path = files.utils.get_base_path(image_field.name)
+        thumbnails_paths = file.thumbnails
+        files.utils.make_thumbnails(
+            abs_path,
+            thumbnails_paths,
+            output_format=THUMBNAIL_FORMAT,
+            quality=THUMBNAIL_QUALITY,
+            optimize=True,
+            progressive=True,
+        )
+        image_field.name = thumbnails_paths[THUMBNAIL_SIZE_DEFAULT]
+        file.save(update_fields={'thumbnail'})
+    # TODO: For a video, source of the thumbnail is some frame fetched with ffpeg
+    elif file.is_video:
+        raise NotImplementedError
+    # TODO: file.thumbnail field is updated

files/templates/files/admin/thumbnails.html

@@ -0,0 +1,14 @@
+<div class="file-thumbnails">
+{% for size, thumb_url in thumbnails.items %}
+  <div class="file-thumbnail">
+    <span class="file-thumbnail-size">{{ size.0 }}x{{ size.1 }}px</span>
+    <img height="{% widthratio size.1 10 1 %}" src="{{ MEDIA_URL }}{{ thumb_url }}" title={{ thumb_url }}>
+  </div>
+{% endfor %}
+{% if thumbnail.width != THUMBNAIL_SIZE_DEFAULT.0 or thumbnail.height != THUMBNAIL_SIZE_DEFAULT.1 %}
+  <p>
+    <b class="icon-alert">⚠</b> Expected {{ THUMBNAIL_SIZE_DEFAULT.0 }}x{{ THUMBNAIL_SIZE_DEFAULT.1 }}px,
+    got {{ thumbnail.width }}x{{ thumbnail.height }}px: resolution of the source file might be too low?
+  </p>
+{% endif %}
+</div>

files/utils.py

@@ -5,14 +5,19 @@ import logging
 import mimetypes
 import os
 import os.path
+import tempfile
 import toml
 import typing
 import zipfile
 
+from PIL import Image
+from django.core.files.storage import default_storage
 from lxml import etree
 import clamd
 import magic
 
+from constants.base import THUMBNAIL_FORMAT, THUMBNAIL_SIZES
+
 logger = logging.getLogger(__name__)
 MODULE_DIR = Path(__file__).resolve().parent
 THEME_SCHEMA = []
@@ -172,3 +177,96 @@ def run_clamdscan(abs_path: str) -> tuple:
         result = clamd_socket.instream(f)['stream']
     logger.info('File at path=%s scanned: %s', abs_path, result)
     return result
+
+
+def get_base_path(file_path: str) -> str:
+    """Return the given path without its extension."""
+    base_path, _ = os.path.splitext(file_path)
+    return base_path
+
+
+def _resize(image: Image, size: tuple, output, output_format: str = 'PNG', **output_params):
+    """Resize a models.ImageField to a given size and write it into output file."""
+    source_image = image.convert('RGBA' if output_format == 'PNG' else 'RGB')
+    source_image.thumbnail(size, Image.LANCZOS)
+    source_image.save(output, output_format, **output_params)
+
+
+def make_thumbnails(image_field, output_paths: dict, output_format: str = 'PNG', **output_params):
+    """
+    Generate thumbnail files for given models.ImageField and list of dimensions.
+
+    Currently only intended to be used manually from shell, e.g.:
+
+        from files.models import File
+        import files.utils
+        files_wo_thumbs = File.objects.filter(type__in=(File.TYPES.IMAGE, File.TYPES.VIDEO))
+        for b in badges:
+            # TODO: thumbnail should be set to THUMBNAIL_SIZE_DEFAULT of the source for IMAGE
+            base_path = files.utils.get_base_path(b.thumbnail.path)
+            files.utils.make_thumbnails(b.thumbnail, base_path, THUMBNAIL_SIZES)
+
+    """
+    thumbnail_ext = output_format.lower()
+    if thumbnail_ext == 'jpeg':
+        thumbnail_ext = 'jpg'
+    thumbnails = {}
+    for (w, h), output_path in output_paths.items():
+        # output_path = f'{base_path}_{w}x{h}.{thumbnail_ext}'
+        with tempfile.TemporaryFile() as f:
+            size = (w, h)
+            logger.info('Resizing %s to %s (%s)', image_field, size, output_format)
+            image = Image.open(image_field)
+            _resize(image, size, f, output_format=output_format, **output_params)
+            logger.info('Saving a new thumbnail to %s', output_path)
+            if default_storage.exists(output_path):
+                logger.warning('%s exists, overwriting', output_path)
+                default_storage.delete(output_path)
+            default_storage.save(output_path, f)
+            thumbnails[size] = output_path
+    return thumbnails
+
+
+def get_thumbnails_paths(file_name: str) -> dict:
+    # Should we check actual existence of the file?
+    # if not storage.exists(file_name):
+    #    return {}
+    base_name, _ = os.path.splitext(file_name)
+    thumbnail_ext = THUMBNAIL_FORMAT.lower()
+    if thumbnail_ext == 'jpeg':
+        thumbnail_ext = 'jpg'
+    return {
+        size_in_px: f'{base_name}_{size_in_px}x{size_in_px}.{thumbnail_ext}'
+        for size_in_px in THUMBNAIL_SIZES
+    }
+
+
+def _delete_file_in_storage(file_name: str) -> None:
+    if not file_name:
+        return
+
+    if not default_storage.exists(file_name):
+        logger.warning("%s doesn't exist in storage, nothing to delete", file_name)
+    else:
+        logger.info('Deleting %s from storage', file_name)
+        default_storage.delete(file_name)
+
+
+def delete_thumbnail_file(file_name: str) -> None:
+    _delete_file_in_storage(file_name)
+
+    # Also delete associated thumbnails
+    thumbnail_names = get_thumbnails_paths(file_name).values()
+    for thumbnail_name in thumbnail_names:
+        if not default_storage.exists(thumbnail_name):
+            continue
+        logger.info(
+            'Deleting thumbnail %s from storage because %s was also deleted',
+            thumbnail_name,
+            file_name,
+        )
+        default_storage.delete(thumbnail_name)
+
+
+def delete_source_file(file_name: str) -> None:
+    _delete_file_in_storage(file_name)