diff --git a/common/storage.py b/common/storage.py index 2345b1b3..86bbb7de 100644 --- a/common/storage.py +++ b/common/storage.py @@ -1,11 +1,16 @@ """Custom file storage classes.""" import logging -from botocore.client import Config from django.conf import settings +from django.db import models +from django.db.models.fields.files import FieldFile + +from botocore.client import Config import boto3 import botocore.exceptions +import nginx_secure_links.storages + from storages.backends.s3boto3 import S3Boto3Storage logger = logging.getLogger(__name__) @@ -49,7 +54,7 @@ def _get_s3_client(): ) -def get_s3_url(path, expires_in_seconds=3600): +def get_s3_url(path, expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS): """Generate a pre-signed S3 URL to a given path.""" global _s3_client if not _s3_client: @@ -94,7 +99,7 @@ def get_s3_post_url_and_fields( bucket=settings.AWS_STORAGE_BUCKET_NAME, fields=None, conditions=None, - expires_in_seconds=3600, + expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS, ): """Generate a presigned URL S3 POST request to upload a file to a given bucket and path. @@ -123,3 +128,36 @@ def get_s3_post_url_and_fields( # The response contains the presigned URL and required fields return response + + +class DynamicStorageFieldFile(FieldFile): + """Defines which storage the file is located at.""" + + def __init__(self, instance, *args, **kwargs): + """Choose between S3 and file system storage depending on `source_storage`.""" + super().__init__(instance, *args, **kwargs) + if instance.source_storage is None: # S3 is default + self.storage = S3Boto3CustomStorage() + elif instance.source_storage == 'fs': + self.storage = nginx_secure_links.storages.FileStorage() + else: + raise + + +class CustomFileField(models.FileField): + """Defines which storage the file field is located at.""" + + attr_class = DynamicStorageFieldFile + + def pre_save(self, model_instance, add): + """Choose between S3 and file system storage depending on `source_storage`.""" + if model_instance.source_storage is None: + storage = S3Boto3CustomStorage() + elif model_instance.source_storage == 'fs': + storage = nginx_secure_links.storages.FileStorage() + else: + raise + self.storage = storage + model_instance.source.storage = storage + # TODO: do the same for thumbnail? + return super().pre_save(model_instance, add) diff --git a/requirements.txt b/requirements.txt index ace813fa..6ab45ddd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ django-background-tasks-updated @ git+https://projects.blender.org/infrastructur django-countries==7.5.1 django-loginas==0.3.11 django-nested-admin==4.0.2 +django-nginx-secure-links==0.0.7 django-pipeline==3.1.0 django-s3direct==2.0.3 django-storages[google]==1.11.1 diff --git a/static_assets/admin.py b/static_assets/admin.py index 912f27c5..a6c00fe3 100644 --- a/static_assets/admin.py +++ b/static_assets/admin.py @@ -66,7 +66,7 @@ class StaticAssetAdmin(AdminUserDefaultMixin, nested_admin.NestedModelAdmin): { 'fields': [ 'id', - 'source', + ('source', 'source_storage'), 'original_filename', 'size_bytes', ('source_type', 'content_type'), diff --git a/static_assets/management/commands/download_to_fs.py b/static_assets/management/commands/download_to_fs.py new file mode 100644 index 00000000..9614e844 --- /dev/null +++ b/static_assets/management/commands/download_to_fs.py @@ -0,0 +1,55 @@ +# noqa: D100 +import logging +import os.path + +from django.core.files.storage import FileSystemStorage +from django.core.management.base import BaseCommand + +from static_assets.models.static_assets import StaticAsset + +file_system_storage = FileSystemStorage() +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class Command(BaseCommand): + """Download static asset files of given IDs to file system storage.""" + + help = "Download static asset files of given IDs to file system storage." + + def add_arguments(self, parser): + """Add range of IDs to command options.""" + parser.add_argument('--min-id', type=int) + parser.add_argument('--mex-id', type=int) + + def handle(self, *args, **options): # noqa: D102 + id_min = options['min_id'] + id_mex = options['mex_id'] + for sa in StaticAsset.objects.filter(id__gte=id_min, id__lt=id_mex).order_by('id'): + # TODO: optionally also update `source_storage` field with `fs` + self._download_to_file_system_storage(sa) + + def _download_to_file_system_storage(self, sa: StaticAsset): + if sa.thumbnail: + self._save(sa.thumbnail, prefix='public') + try: + video = sa.video + for variation in video.variations.all(): + self._save(variation.source) + for track in video.tracks.all(): + self._save(track.source) + except StaticAsset.video.RelatedObjectDoesNotExist: + pass + # sa.image has no extra files + self._save(sa.source) + + def _save(self, field, prefix=''): + output_path = os.path.join(prefix, field.name) + logger.info('Downloading %s to path %s', field, output_path) + if file_system_storage.exists(output_path): + logger.warning('%s exists', output_path) + return + f = field.open() + file_system_storage.save(output_path, f) + f.close() + logger.info('Downloaded %s to path %s', field, output_path) diff --git a/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py b/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py new file mode 100644 index 00000000..fbee55e2 --- /dev/null +++ b/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py @@ -0,0 +1,57 @@ +# Generated by Django 4.2.13 on 2024-07-09 10:06 + +import common.storage +import common.upload_paths +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('static_assets', '0012_allow_blank_license'), + ] + + operations = [ + migrations.AddField( + model_name='staticasset', + name='source_storage', + field=models.CharField( + blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True + ), + ), + migrations.AddField( + model_name='videotrack', + name='source_storage', + field=models.CharField( + blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True + ), + ), + migrations.AddField( + model_name='videovariation', + name='source_storage', + field=models.CharField( + blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True + ), + ), + migrations.AlterField( + model_name='staticasset', + name='source', + field=common.storage.CustomFileField( + blank=True, max_length=256, upload_to=common.upload_paths.get_upload_to_hashed_path + ), + ), + migrations.AlterField( + model_name='videotrack', + name='source', + field=common.storage.CustomFileField( + blank=True, max_length=256, upload_to=common.upload_paths.get_upload_to_hashed_path + ), + ), + migrations.AlterField( + model_name='videovariation', + name='source', + field=common.storage.CustomFileField( + blank=True, max_length=256, upload_to=common.upload_paths.get_upload_to_hashed_path + ), + ), + ] diff --git a/static_assets/models/static_assets.py b/static_assets/models/static_assets.py index 6f5aecf4..66840631 100644 --- a/static_assets/models/static_assets.py +++ b/static_assets/models/static_assets.py @@ -14,14 +14,20 @@ from django.utils.text import slugify import looper.model_mixins from common import mixins +from common.storage import CustomFileField from common.upload_paths import get_upload_to_hashed_path from static_assets.models import License from static_assets.tasks import create_video_processing_job, create_video_transcribing_job -import common.storage User = get_user_model() log = logging.getLogger(__name__) +STORAGE_CHOICES = [ + (None, 'S3'), + ('fs', 'File System'), + ('fsp', 'File System Public'), +] + def _get_default_license_id() -> Optional[int]: cc_by = License.objects.filter(slug='cc-by').first() @@ -44,12 +50,12 @@ class StaticAsset( class Meta: ordering = ['-date_created'] - source = models.FileField( + source = CustomFileField( upload_to=get_upload_to_hashed_path, - storage=common.storage.S3Boto3CustomStorage(), blank=True, max_length=256, ) + source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES) source_type = models.CharField( choices=StaticAssetFileTypeChoices.choices, max_length=5, @@ -316,7 +322,8 @@ class VideoVariation(models.Model): height = models.PositiveIntegerField(blank=True, null=True) width = models.PositiveIntegerField(blank=True, null=True) resolution_label = models.CharField(max_length=32, blank=True) - source = models.FileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES) size_bytes = models.BigIntegerField(editable=False) content_type = models.CharField(max_length=256, blank=True) @@ -356,7 +363,8 @@ class VideoTrack(models.Model): language = models.CharField( blank=False, null=False, max_length=5, choices=VideoTrackLanguageCodeChoices.choices ) - source = models.FileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES) @property def url(self) -> str: diff --git a/studio/settings.py b/studio/settings.py index 7ca09663..61e30e23 100644 --- a/studio/settings.py +++ b/studio/settings.py @@ -75,6 +75,7 @@ INSTALLED_APPS = [ 'rest_framework', 'rest_framework.authtoken', 's3direct', + 'nginx_secure_links', ] AUTH_USER_MODEL = 'users.User' @@ -335,7 +336,6 @@ SITE_ID = 1 # Required by Django Debug Toolbar INTERNAL_IPS = ['127.0.0.1'] - TAGGIT_CASE_INSENSITIVE = True DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' @@ -692,3 +692,8 @@ STRIPE_CHECKOUT_SUBMIT_TYPE = 'pay' # Maximum number of attempts for failing background tasks MAX_ATTEMPTS = 3 + +FILE_LINK_EXPIRE_SECONDS = 3600 +SECURE_LINK_SECRET_KEY = _get('SECURE_LINK_SECRET_KEY') +SECURE_LINK_EXPIRATION_SECONDS = FILE_LINK_EXPIRE_SECONDS +SECURE_LINK_PUBLIC_PREFIXES = ['public']