WIP: Switch between FS storage and S3 per static asset #104417
@ -2,7 +2,6 @@
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.files.storage import FileSystemStorage
|
||||
from django.db import models
|
||||
from django.db.models.fields.files import FieldFile
|
||||
|
||||
@ -10,6 +9,8 @@ from botocore.client import Config
|
||||
import boto3
|
||||
import botocore.exceptions
|
||||
|
||||
import nginx_secure_links.storages
|
||||
|
||||
from storages.backends.s3boto3 import S3Boto3Storage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -53,7 +54,7 @@ def _get_s3_client():
|
||||
)
|
||||
|
||||
|
||||
def get_s3_url(path, expires_in_seconds=3600):
|
||||
def get_s3_url(path, expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS):
|
||||
"""Generate a pre-signed S3 URL to a given path."""
|
||||
global _s3_client
|
||||
if not _s3_client:
|
||||
@ -98,7 +99,7 @@ def get_s3_post_url_and_fields(
|
||||
bucket=settings.AWS_STORAGE_BUCKET_NAME,
|
||||
fields=None,
|
||||
conditions=None,
|
||||
expires_in_seconds=3600,
|
||||
expires_in_seconds=settings.FILE_LINK_EXPIRE_SECONDS,
|
||||
):
|
||||
"""Generate a presigned URL S3 POST request to upload a file to a given bucket and path.
|
||||
|
||||
@ -138,7 +139,7 @@ class DynamicStorageFieldFile(FieldFile):
|
||||
if instance.source_storage is None: # S3 is default
|
||||
self.storage = S3Boto3CustomStorage()
|
||||
elif instance.source_storage == 'fs':
|
||||
self.storage = FileSystemStorage()
|
||||
self.storage = nginx_secure_links.storages.FileStorage()
|
||||
else:
|
||||
raise
|
||||
|
||||
@ -153,7 +154,7 @@ class CustomFileField(models.FileField):
|
||||
if model_instance.source_storage is None:
|
||||
storage = S3Boto3CustomStorage()
|
||||
elif model_instance.source_storage == 'fs':
|
||||
storage = FileSystemStorage()
|
||||
storage = nginx_secure_links.storages.FileStorage()
|
||||
else:
|
||||
raise
|
||||
self.storage = storage
|
||||
|
@ -29,6 +29,7 @@ django-background-tasks-updated @ git+https://projects.blender.org/infrastructur
|
||||
django-countries==7.5.1
|
||||
django-loginas==0.3.11
|
||||
django-nested-admin==4.0.2
|
||||
django-nginx-secure-links==0.0.7
|
||||
django-pipeline==3.1.0
|
||||
django-s3direct==2.0.3
|
||||
django-storages[google]==1.11.1
|
||||
|
@ -31,7 +31,7 @@ class Command(BaseCommand):
|
||||
|
||||
def _download_to_file_system_storage(self, sa: StaticAsset):
|
||||
if sa.thumbnail:
|
||||
self._save(sa.thumbnail, prefix='thumbnails')
|
||||
self._save(sa.thumbnail, prefix='public')
|
||||
try:
|
||||
video = sa.video
|
||||
for variation in video.variations.all():
|
||||
|
@ -16,21 +16,21 @@ class Migration(migrations.Migration):
|
||||
model_name='staticasset',
|
||||
name='source_storage',
|
||||
field=models.CharField(
|
||||
blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True
|
||||
blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='videotrack',
|
||||
name='source_storage',
|
||||
field=models.CharField(
|
||||
blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True
|
||||
blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='videovariation',
|
||||
name='source_storage',
|
||||
field=models.CharField(
|
||||
blank=True, choices=[(None, 'S3'), ('fs', 'File System')], max_length=3, null=True
|
||||
blank=True, choices=[(None, 'S3'), ('fs', 'File System'), ('fsp', 'File System Public')], max_length=3, null=True
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
|
@ -22,6 +22,12 @@ from static_assets.tasks import create_video_processing_job, create_video_transc
|
||||
User = get_user_model()
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STORAGE_CHOICES = [
|
||||
(None, 'S3'),
|
||||
('fs', 'File System'),
|
||||
('fsp', 'File System Public'),
|
||||
]
|
||||
|
||||
|
||||
def _get_default_license_id() -> Optional[int]:
|
||||
cc_by = License.objects.filter(slug='cc-by').first()
|
||||
@ -49,12 +55,7 @@ class StaticAsset(
|
||||
blank=True,
|
||||
max_length=256,
|
||||
)
|
||||
source_storage = models.CharField(
|
||||
max_length=3,
|
||||
null=True,
|
||||
blank=True,
|
||||
choices=[(None, 'S3'), ('fs', 'File System')],
|
||||
)
|
||||
source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES)
|
||||
source_type = models.CharField(
|
||||
choices=StaticAssetFileTypeChoices.choices,
|
||||
max_length=5,
|
||||
@ -322,12 +323,7 @@ class VideoVariation(models.Model):
|
||||
width = models.PositiveIntegerField(blank=True, null=True)
|
||||
resolution_label = models.CharField(max_length=32, blank=True)
|
||||
source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256)
|
||||
source_storage = models.CharField(
|
||||
max_length=3,
|
||||
null=True,
|
||||
blank=True,
|
||||
choices=[(None, 'S3'), ('fs', 'File System')],
|
||||
)
|
||||
source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES)
|
||||
size_bytes = models.BigIntegerField(editable=False)
|
||||
content_type = models.CharField(max_length=256, blank=True)
|
||||
|
||||
@ -368,12 +364,7 @@ class VideoTrack(models.Model):
|
||||
blank=False, null=False, max_length=5, choices=VideoTrackLanguageCodeChoices.choices
|
||||
)
|
||||
source = CustomFileField(upload_to=get_upload_to_hashed_path, blank=True, max_length=256)
|
||||
source_storage = models.CharField(
|
||||
max_length=3,
|
||||
null=True,
|
||||
blank=True,
|
||||
choices=[(None, 'S3'), ('fs', 'File System')],
|
||||
)
|
||||
source_storage = models.CharField(max_length=3, null=True, blank=True, choices=STORAGE_CHOICES)
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
|
@ -75,6 +75,7 @@ INSTALLED_APPS = [
|
||||
'rest_framework',
|
||||
'rest_framework.authtoken',
|
||||
's3direct',
|
||||
'nginx_secure_links',
|
||||
]
|
||||
|
||||
AUTH_USER_MODEL = 'users.User'
|
||||
@ -335,7 +336,6 @@ SITE_ID = 1
|
||||
# Required by Django Debug Toolbar
|
||||
INTERNAL_IPS = ['127.0.0.1']
|
||||
|
||||
|
||||
TAGGIT_CASE_INSENSITIVE = True
|
||||
|
||||
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
|
||||
@ -692,3 +692,8 @@ STRIPE_CHECKOUT_SUBMIT_TYPE = 'pay'
|
||||
|
||||
# Maximum number of attempts for failing background tasks
|
||||
MAX_ATTEMPTS = 3
|
||||
|
||||
FILE_LINK_EXPIRE_SECONDS = 3600
|
||||
SECURE_LINK_SECRET_KEY = _get('SECURE_LINK_SECRET_KEY')
|
||||
SECURE_LINK_EXPIRATION_SECONDS = FILE_LINK_EXPIRE_SECONDS
|
||||
SECURE_LINK_PUBLIC_PREFIXES = ['public']
|
||||
|
Loading…
Reference in New Issue
Block a user