From 5dec29221155fdc0f164730bb3edc1436686ac02 Mon Sep 17 00:00:00 2001 From: Anna Sirota Date: Tue, 9 Jul 2024 11:23:47 +0200 Subject: [PATCH 1/3] Re-pin all deps with `pip freeze` --- requirements.txt | 33 ++++++++++++++++++--------------- requirements_dev.txt | 10 +++++----- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index d5bccea1..ace813fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,13 @@ -alphabetic-timestamp==1.1.5 +## The following requirements were added by pip freeze: +alphabetic_timestamp==1.1.5 appdirs==1.4.4 arabic-reshaper==3.0.0 asgiref==3.8.1 asn1crypto==1.5.1 attrs==19.3.0 -babel==2.12.1 +Babel==2.12.1 bleach==3.3.1 -blender-id-oauth-client @ git+https://projects.blender.org/infrastructure/blender-id-oauth-client.git@452646e +blender-id-oauth-client @ git+https://projects.blender.org/infrastructure/blender-id-oauth-client.git@452646e0742b544494f3e48ad60f5774522f4f92 boto3==1.34.130 botocore==1.34.130 braintree==4.17.1 @@ -21,9 +22,10 @@ cryptography==42.0.8 cssselect2==0.7.0 defusedxml==0.7.1 dj-database-url==1.0.0 +Django==4.2.13 django-activity-stream==2.0.0 django-anymail[mailgun]==8.2 -django-background-tasks-updated @ git+https://projects.blender.org/infrastructure/django-background-tasks.git@98508d6 +django-background-tasks-updated @ git+https://projects.blender.org/infrastructure/django-background-tasks.git@98508d66905997925f9db399222f2264310c8e37 django-countries==7.5.1 django-loginas==0.3.11 django-nested-admin==4.0.2 @@ -31,7 +33,6 @@ django-pipeline==3.1.0 django-s3direct==2.0.3 django-storages[google]==1.11.1 django-taggit==5.0.1 -django==4.2.13 djangorestframework==3.14.0 filelock==3.8.0 geoip2==3.0.0 @@ -46,16 +47,16 @@ html5lib==1.1 idna==3.4 isodate==0.6.1 itsdangerous==2.1.2 -jinja2==2.11.3 +Jinja2==2.11.3 jmespath==0.10.0 jsmin==3.0.1 libsass==0.22.0 libsasscompiler==0.1.9 localflavor==1.9 -looper @ git+https://projects.blender.org/infrastructure/looper.git@da42680 +looper @ git+https://projects.blender.org/infrastructure/looper.git@da426807b727c6779445b40e28d2f8a447f3a1dd lxml==4.9.2 -markdown==3.4.1 -markupsafe==1.1.1 +Markdown==3.4.1 +MarkupSafe==1.1.1 maxminddb==2.2.0 meilisearch==0.18.3 mistune==2.0.0a4 @@ -65,13 +66,15 @@ packaging==23.0 pillow==10.3.0 protobuf==4.21.9 psycopg2==2.9.5 -pyasn1-modules==0.2.8 pyasn1==0.4.8 +pyasn1-modules==0.2.8 pycountry==22.3.5 +pycparser==2.22 +pyHanko==0.25.0 pyhanko-certvalidator==0.26.3 -pyhanko==0.25.0 pyinstrument==4.5.3 pymongo==3.13.0 +pyOpenSSL==24.1.0 pyparsing==3.0.9 pypdf==4.2.0 pypng==0.20220715.0 @@ -82,17 +85,16 @@ python-monkey-business==1.0.0 python-stdnum==1.18 pytz==2022.7.1 pyvat @ git+https://github.com/iconfinder/pyvat.git@419abd659ae5a4a6cb6ea9b54aa4bde17aefeb5b -pyyaml==6.0 +PyYAML==6.0 qrcode==7.4.2 reportlab==4.2.0 +requests==2.32.3 requests-file==1.5.1 requests-oauthlib==1.3.1 requests-toolbelt==0.10.1 -requests==2.32.3 rsa==4.9 s3transfer==0.10.1 sentry-sdk==1.16.0 -setuptools==68.2.2 shortcodes==2.5.0 six==1.16.0 sorl-thumbnail==12.10.0 @@ -101,7 +103,8 @@ stripe==7.1.0 svglib==1.5.1 tinycss2==1.3.0 tldextract==3.4.0 -typing-extensions==4.12.1 +typing_extensions==4.12.1 +tzdata==2024.1 tzlocal==5.2 uritools==4.0.3 urllib3==1.26.14 diff --git a/requirements_dev.txt b/requirements_dev.txt index 579aee7e..69380049 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -10,17 +10,17 @@ django-stubs-ext==0.7.0 django-stubs==1.13.0 djhtml==1.4.0 factory-boy==3.2.1 -faker==15.3.1 -flake8-docstrings==1.6.0 +Faker==15.3.1 flake8==3.9.2 +flake8-docstrings==1.6.0 freezegun==1.2.2 identify==2.5.8 ipython==7.34.0 jedi==0.18.1 matplotlib-inline==0.1.6 mccabe==0.6.1 -mypy-extensions==0.4.3 mypy==0.990 +mypy-extensions==0.4.3 nodeenv==1.7.0 parso==0.8.3 pathspec==0.10.1 @@ -33,7 +33,7 @@ ptyprocess==0.7.0 pycodestyle==2.7.0 pydocstyle==6.1.1 pyflakes==2.3.1 -pygments==2.13.0 +Pygments==2.13.0 responses==0.25.3 snowballstemmer==2.2.0 tblib==3.0.0 @@ -41,6 +41,6 @@ toml==0.10.2 tomli==2.0.1 traitlets==5.5.0 types-pytz==2022.6.0.1 -types-pyyaml==6.0.12.2 +types-PyYAML==6.0.12.2 virtualenv==20.16.6 wcwidth==0.2.5 -- 2.30.2 From 672b3bf7f3447deb6be730f130916ecd7da186d1 Mon Sep 17 00:00:00 2001 From: Anna Sirota Date: Tue, 9 Jul 2024 12:48:29 +0200 Subject: [PATCH 2/3] Allow switching between FS and S3 storages per static asset --- common/storage.py | 39 ++++++++++++- static_assets/admin.py | 2 +- .../management/commands/download_to_fs.py | 55 ++++++++++++++++++ ...rage_videotrack_source_storage_and_more.py | 57 +++++++++++++++++++ static_assets/models/static_assets.py | 27 +++++++-- 5 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 static_assets/management/commands/download_to_fs.py create mode 100644 static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py diff --git a/common/storage.py b/common/storage.py index 2345b1b3..ba52b8c5 100644 --- a/common/storage.py +++ b/common/storage.py @@ -1,8 +1,12 @@ """Custom file storage classes.""" import logging -from botocore.client import Config from django.conf import settings +from django.core.files.storage import FileSystemStorage +from django.db import models +from django.db.models.fields.files import FieldFile + +from botocore.client import Config import boto3 import botocore.exceptions @@ -123,3 +127,36 @@ def get_s3_post_url_and_fields( # The response contains the presigned URL and required fields return response + + +class DynamicStorageFieldFile(FieldFile): + """Defines which storage the file is located at.""" + + def __init__(self, instance, *args, **kwargs): + """Choose between S3 and file system storage depending on `source_storage`.""" + super().__init__(instance, *args, **kwargs) + if instance.source_storage is None: # S3 is default + self.storage = S3Boto3CustomStorage() + elif instance.source_storage == 'fs': + self.storage = FileSystemStorage() + else: + raise + + +class CustomFileField(models.FileField): + """Defines which storage the file field is located at.""" + + attr_class = DynamicStorageFieldFile + + def pre_save(self, model_instance, add): + """Choose between S3 and file system storage depending on `source_storage`.""" + if model_instance.source_storage is None: + storage = S3Boto3CustomStorage() + elif model_instance.source_storage == 'fs': + storage = FileSystemStorage() + else: + raise + self.storage = storage + model_instance.source.storage = storage + # TODO: do the same for thumbnail? + return super().pre_save(model_instance, add) diff --git a/static_assets/admin.py b/static_assets/admin.py index 912f27c5..a6c00fe3 100644 --- a/static_assets/admin.py +++ b/static_assets/admin.py @@ -66,7 +66,7 @@ class StaticAssetAdmin(AdminUserDefaultMixin, nested_admin.NestedModelAdmin): { 'fields': [ 'id', - 'source', + ('source', 'source_storage'), 'original_filename', 'size_bytes', ('source_type', 'content_type'), diff --git a/static_assets/management/commands/download_to_fs.py b/static_assets/management/commands/download_to_fs.py new file mode 100644 index 00000000..e8b2f354 --- /dev/null +++ b/static_assets/management/commands/download_to_fs.py @@ -0,0 +1,55 @@ +# noqa: D100 +import logging +import os.path + +from django.core.files.storage import FileSystemStorage +from django.core.management.base import BaseCommand + +from static_assets.models.static_assets import StaticAsset + +file_system_storage = FileSystemStorage() +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class Command(BaseCommand): + """Download static asset files of given IDs to file system storage.""" + + help = "Download static asset files of given IDs to file system storage." + + def add_arguments(self, parser): + """Add range of IDs to command options.""" + parser.add_argument('--min-id', type=int) + parser.add_argument('--mex-id', type=int) + + def handle(self, *args, **options): # noqa: D102 + id_min = options['min_id'] + id_mex = options['mex_id'] + for sa in StaticAsset.objects.filter(id__gte=id_min, id__lt=id_mex).order_by('id'): + # TODO: optionally also update `source_storage` field with `fs` + self._download_to_file_system_storage(sa) + + def _download_to_file_system_storage(self, sa: StaticAsset): + if sa.thumbnail: + self._save(sa.thumbnail, prefix='thumbnails') + try: + video = sa.video + for variation in video.variations.all(): + self._save(variation.source) + for track in video.tracks.all(): + self._save(track.source) + except StaticAsset.video.RelatedObjectDoesNotExist: + pass + # sa.image has no extra files + self._save(sa.source) + + def _save(self, field, prefix=''): + output_path = os.path.join(prefix, field.name) + logger.info('Downloading %s to path %s', field, output_path) + if file_system_storage.exists(output_path): + logger.warning('%s exists', output_path) + return + f = field.open() + file_system_storage.save(output_path, f) + f.close() + logger.info('Downloaded %s to path %s', field, output_path) diff --git a/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py b/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py new file mode 100644 index 00000000..13688c6f --- /dev/null +++ b/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py @@ -0,0 +1,57 @@ +# Generated by Django 4.2.13 on 2024-07-09 10:06 + +import common.storage +import common.upload_paths +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('static_assets', '0012_allow_blank_license'), + ] + + operations = [ + migrations.AddField( + model_name='staticasset', + name='source_storage', + field=models.CharField( + blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True + ), + ), + migrations.AddField( + model_name='videotrack', + name='source_storage', + field=models.CharField( + blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True + ), + ), + migrations.AddField( + model_name='videovariation', + name='source_storage', + field=models.CharField( + blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True + ), + ), + migrations.AlterField( + model_name='staticasset', + name='source', + field=common.storage.CustomFileField( + blank=True, max_length=256, upload_to=common.upload_paths.get_upload_to_hashed_path + ), + ), + migrations.AlterField( + model_name='videotrack', + name='source', + field=common.storage.CustomFileField( + blank=True, max_length=256, upload_to=common.upload_paths.get_upload_to_hashed_path + ), + ), + migrations.AlterField( + model_name='videovariation', + name='source', + field=common.storage.CustomFileField( + blank=True, max_length=256, upload_to=common.upload_paths.get_upload_to_hashed_path + ), + ), + ] diff --git a/static_assets/models/static_assets.py b/static_assets/models/static_assets.py index 6f5aecf4..61d9293d 100644 --- a/static_assets/models/static_assets.py +++ b/static_assets/models/static_assets.py @@ -14,10 +14,10 @@ from django.utils.text import slugify import looper.model_mixins from common import mixins +from common.storage import CustomFileField from common.upload_paths import get_upload_to_hashed_path from static_assets.models import License from static_assets.tasks import create_video_processing_job, create_video_transcribing_job -import common.storage User = get_user_model() log = logging.getLogger(__name__) @@ -44,12 +44,17 @@ class StaticAsset( class Meta: ordering = ['-date_created'] - source = models.FileField( + source = CustomFileField( upload_to=get_upload_to_hashed_path, - storage=common.storage.S3Boto3CustomStorage(), blank=True, max_length=256, ) + source_storage = models.CharField( + max_length=3, + null=True, + blank=True, + choices=[(None, 'S3'), ('fs', 'File System')], + ) source_type = models.CharField( choices=StaticAssetFileTypeChoices.choices, max_length=5, @@ -316,7 +321,13 @@ class VideoVariation(models.Model): height = models.PositiveIntegerField(blank=True, null=True) width = models.PositiveIntegerField(blank=True, null=True) resolution_label = models.CharField(max_length=32, blank=True) - source = models.FileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source_storage = models.CharField( + max_length=3, + null=True, + blank=True, + choices=[(None, 'S3'), ('fs', 'File System')], + ) size_bytes = models.BigIntegerField(editable=False) content_type = models.CharField(max_length=256, blank=True) @@ -356,7 +367,13 @@ class VideoTrack(models.Model): language = models.CharField( blank=False, null=False, max_length=5, choices=VideoTrackLanguageCodeChoices.choices ) - source = models.FileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) + source_storage = models.CharField( + max_length=3, + null=True, + blank=True, + choices=[(None, 'S3'), ('fs', 'File System')], + ) @property def url(self) -> str: -- 2.30.2 From 189c30b8eec46f553dbeea71e63ab70e4cb90a73 Mon Sep 17 00:00:00 2001 From: Anna Sirota Date: Tue, 9 Jul 2024 18:14:43 +0200 Subject: [PATCH 3/3] Configure nginx signing --- common/storage.py | 11 ++++---- requirements.txt | 1 + .../management/commands/download_to_fs.py | 2 +- ...rage_videotrack_source_storage_and_more.py | 6 ++--- static_assets/models/static_assets.py | 27 +++++++------------ studio/settings.py | 7 ++++- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/common/storage.py b/common/storage.py index ba52b8c5..86bbb7de 100644 --- a/common/storage.py +++ b/common/storage.py @@ -2,7 +2,6 @@ import logging from django.conf import settings -from django.core.files.storage import FileSystemStorage from django.db import models from django.db.models.fields.files import FieldFile @@ -10,6 +9,8 @@ from botocore.client import Config import boto3 import botocore.exceptions +import nginx_secure_links.storages + from storages.backends.s3boto3 import S3Boto3Storage logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def _get_s3_client(): ) -def get_s3_url(path, expires_in_seconds=3600): +def get_s3_url(path, expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS): """Generate a pre-signed S3 URL to a given path.""" global _s3_client if not _s3_client: @@ -98,7 +99,7 @@ def get_s3_post_url_and_fields( bucket=settings.AWS_STORAGE_BUCKET_NAME, fields=None, conditions=None, - expires_in_seconds=3600, + expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS, ): """Generate a presigned URL S3 POST request to upload a file to a given bucket and path. @@ -138,7 +139,7 @@ class DynamicStorageFieldFile(FieldFile): if instance.source_storage is None: # S3 is default self.storage = S3Boto3CustomStorage() elif instance.source_storage == 'fs': - self.storage = FileSystemStorage() + self.storage = nginx_secure_links.storages.FileStorage() else: raise @@ -153,7 +154,7 @@ class CustomFileField(models.FileField): if model_instance.source_storage is None: storage = S3Boto3CustomStorage() elif model_instance.source_storage == 'fs': - storage = FileSystemStorage() + storage = nginx_secure_links.storages.FileStorage() else: raise self.storage = storage diff --git a/requirements.txt b/requirements.txt index ace813fa..6ab45ddd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ django-background-tasks-updated @ git+https://projects.blender.org/infrastructur django-countries==7.5.1 django-loginas==0.3.11 django-nested-admin==4.0.2 +django-nginx-secure-links==0.0.7 django-pipeline==3.1.0 django-s3direct==2.0.3 django-storages[google]==1.11.1 diff --git a/static_assets/management/commands/download_to_fs.py b/static_assets/management/commands/download_to_fs.py index e8b2f354..9614e844 100644 --- a/static_assets/management/commands/download_to_fs.py +++ b/static_assets/management/commands/download_to_fs.py @@ -31,7 +31,7 @@ class Command(BaseCommand): def _download_to_file_system_storage(self, sa: StaticAsset): if sa.thumbnail: - self._save(sa.thumbnail, prefix='thumbnails') + self._save(sa.thumbnail, prefix='public') try: video = sa.video for variation in video.variations.all(): diff --git a/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py b/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py index 13688c6f..fbee55e2 100644 --- a/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py +++ b/static_assets/migrations/0013_staticasset_source_storage_videotrack_source_storage_and_more.py @@ -16,21 +16,21 @@ class Migration(migrations.Migration): model_name='staticasset', name='source_storage', field=models.CharField( - blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True + blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True ), ), migrations.AddField( model_name='videotrack', name='source_storage', field=models.CharField( - blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True + blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True ), ), migrations.AddField( model_name='videovariation', name='source_storage', field=models.CharField( - blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True + blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True ), ), migrations.AlterField( diff --git a/static_assets/models/static_assets.py b/static_assets/models/static_assets.py index 61d9293d..66840631 100644 --- a/static_assets/models/static_assets.py +++ b/static_assets/models/static_assets.py @@ -22,6 +22,12 @@ from static_assets.tasks import create_video_processing_job, create_video_transc User = get_user_model() log = logging.getLogger(__name__) +STORAGE_CHOICES = [ + (None, 'S3'), + ('fs', 'File System'), + ('fsp', 'File System Public'), +] + def _get_default_license_id() -> Optional[int]: cc_by = License.objects.filter(slug='cc-by').first() @@ -49,12 +55,7 @@ class StaticAsset( blank=True, max_length=256, ) - source_storage = models.CharField( - max_length=3, - null=True, - blank=True, - choices=[(None, 'S3'), ('fs', 'File System')], - ) + source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES) source_type = models.CharField( choices=StaticAssetFileTypeChoices.choices, max_length=5, @@ -322,12 +323,7 @@ class VideoVariation(models.Model): width = models.PositiveIntegerField(blank=True, null=True) resolution_label = models.CharField(max_length=32, blank=True) source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) - source_storage = models.CharField( - max_length=3, - null=True, - blank=True, - choices=[(None, 'S3'), ('fs', 'File System')], - ) + source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES) size_bytes = models.BigIntegerField(editable=False) content_type = models.CharField(max_length=256, blank=True) @@ -368,12 +364,7 @@ class VideoTrack(models.Model): blank=False, null=False, max_length=5, choices=VideoTrackLanguageCodeChoices.choices ) source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256) - source_storage = models.CharField( - max_length=3, - null=True, - blank=True, - choices=[(None, 'S3'), ('fs', 'File System')], - ) + source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES) @property def url(self) -> str: diff --git a/studio/settings.py b/studio/settings.py index 7ca09663..61e30e23 100644 --- a/studio/settings.py +++ b/studio/settings.py @@ -75,6 +75,7 @@ INSTALLED_APPS = [ 'rest_framework', 'rest_framework.authtoken', 's3direct', + 'nginx_secure_links', ] AUTH_USER_MODEL = 'users.User' @@ -335,7 +336,6 @@ SITE_ID = 1 # Required by Django Debug Toolbar INTERNAL_IPS = ['127.0.0.1'] - TAGGIT_CASE_INSENSITIVE = True DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' @@ -692,3 +692,8 @@ STRIPE_CHECKOUT_SUBMIT_TYPE = 'pay' # Maximum number of attempts for failing background tasks MAX_ATTEMPTS = 3 + +FILE_LINK_EXPIRE_SECONDS = 3600 +SECURE_LINK_SECRET_KEY = _get('SECURE_LINK_SECRET_KEY') +SECURE_LINK_EXPIRATION_SECONDS = FILE_LINK_EXPIRE_SECONDS +SECURE_LINK_PUBLIC_PREFIXES = ['public'] -- 2.30.2