WIP: Switch between FS storage and S3 per static asset #104417

Draft
Anna Sirota wants to merge 4 commits from alternative-storage into main

When changing the target branch, be careful to rebase the branch in your fork to match. See documentation.
6 changed files with 26 additions and 28 deletions
Showing only changes of commit 189c30b8ee - Show all commits

View File

@ -2,7 +2,6 @@
import logging
from django.conf import settings
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.db.models.fields.files import FieldFile
@ -10,6 +9,8 @@ from botocore.client import Config
import boto3
import botocore.exceptions
import nginx_secure_links.storages
from storages.backends.s3boto3 import S3Boto3Storage
logger = logging.getLogger(__name__)
@ -53,7 +54,7 @@ def _get_s3_client():
)
def get_s3_url(path, expires_in_seconds=3600):
def get_s3_url(path, expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS):
"""Generate a pre-signed S3 URL to a given path."""
global _s3_client
if not _s3_client:
@ -98,7 +99,7 @@ def get_s3_post_url_and_fields(
bucket=settings.AWS_STORAGE_BUCKET_NAME,
fields=None,
conditions=None,
expires_in_seconds=3600,
expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS,
):
"""Generate a presigned URL S3 POST request to upload a file to a given bucket and path.
@ -138,7 +139,7 @@ class DynamicStorageFieldFile(FieldFile):
if instance.source_storage is None: # S3 is default
self.storage = S3Boto3CustomStorage()
elif instance.source_storage == 'fs':
self.storage = FileSystemStorage()
self.storage = nginx_secure_links.storages.FileStorage()
else:
raise
@ -153,7 +154,7 @@ class CustomFileField(models.FileField):
if model_instance.source_storage is None:
storage = S3Boto3CustomStorage()
elif model_instance.source_storage == 'fs':
storage = FileSystemStorage()
storage = nginx_secure_links.storages.FileStorage()
else:
raise
self.storage = storage

View File

@ -29,6 +29,7 @@ django-background-tasks-updated @ git+https://projects.blender.org/infrastructur
django-countries==7.5.1
django-loginas==0.3.11
django-nested-admin==4.0.2
django-nginx-secure-links==0.0.7
django-pipeline==3.1.0
django-s3direct==2.0.3
django-storages[google]==1.11.1

View File

@ -31,7 +31,7 @@ class Command(BaseCommand):
def _download_to_file_system_storage(self, sa: StaticAsset):
if sa.thumbnail:
self._save(sa.thumbnail, prefix='thumbnails')
self._save(sa.thumbnail, prefix='public')
try:
video = sa.video
for variation in video.variations.all():

View File

@ -16,21 +16,21 @@ class Migration(migrations.Migration):
model_name='staticasset',
name='source_storage',
field=models.CharField(
blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True
blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True
),
),
migrations.AddField(
model_name='videotrack',
name='source_storage',
field=models.CharField(
blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True
blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True
),
),
migrations.AddField(
model_name='videovariation',
name='source_storage',
field=models.CharField(
blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True
blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True
),
),
migrations.AlterField(

View File

@ -22,6 +22,12 @@ from static_assets.tasks import create_video_processing_job, create_video_transc
User = get_user_model()
log = logging.getLogger(__name__)
STORAGE_CHOICES = [
(None, 'S3'),
('fs', 'File System'),
('fsp', 'File System Public'),
]
def _get_default_license_id() -> Optional[int]:
cc_by = License.objects.filter(slug='cc-by').first()
@ -49,12 +55,7 @@ class StaticAsset(
blank=True,
max_length=256,
)
source_storage = models.CharField(
max_length=3,
null=True,
blank=True,
choices=[(None, 'S3'), ('fs', 'File System')],
)
source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES)
source_type = models.CharField(
choices=StaticAssetFileTypeChoices.choices,
max_length=5,
@ -322,12 +323,7 @@ class VideoVariation(models.Model):
width = models.PositiveIntegerField(blank=True, null=True)
resolution_label = models.CharField(max_length=32, blank=True)
source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256)
source_storage = models.CharField(
max_length=3,
null=True,
blank=True,
choices=[(None, 'S3'), ('fs', 'File System')],
)
source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES)
size_bytes = models.BigIntegerField(editable=False)
content_type = models.CharField(max_length=256, blank=True)
@ -368,12 +364,7 @@ class VideoTrack(models.Model):
blank=False, null=False, max_length=5, choices=VideoTrackLanguageCodeChoices.choices
)
source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256)
source_storage = models.CharField(
max_length=3,
null=True,
blank=True,
choices=[(None, 'S3'), ('fs', 'File System')],
)
source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES)
@property
def url(self) -> str:

View File

@ -75,6 +75,7 @@ INSTALLED_APPS = [
'rest_framework',
'rest_framework.authtoken',
's3direct',
'nginx_secure_links',
]
AUTH_USER_MODEL = 'users.User'
@ -335,7 +336,6 @@ SITE_ID = 1
# Required by Django Debug Toolbar
INTERNAL_IPS = ['127.0.0.1']
TAGGIT_CASE_INSENSITIVE = True
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
@ -692,3 +692,8 @@ STRIPE_CHECKOUT_SUBMIT_TYPE = 'pay'
# Maximum number of attempts for failing background tasks
MAX_ATTEMPTS = 3
FILE_LINK_EXPIRE_SECONDS = 3600
SECURE_LINK_SECRET_KEY = _get('SECURE_LINK_SECRET_KEY')
SECURE_LINK_EXPIRATION_SECONDS = FILE_LINK_EXPIRE_SECONDS
SECURE_LINK_PUBLIC_PREFIXES = ['public']