Support for platforms and wheels #131

Merged
Oleg-Komarov merged 3 commits from platforms into main 2024-05-16 18:20:44 +02:00
15 changed files with 271 additions and 53 deletions

View File

@ -6,7 +6,7 @@ from mdgen import MarkdownPostProvider
import factory import factory
import factory.fuzzy import factory.fuzzy
from extensions.models import Extension, Version, Tag, Preview from extensions.models import Extension, Version, Tag, Preview, Platform
from ratings.models import Rating from ratings.models import Rating
fake_markdown = Faker() fake_markdown = Faker()
@ -83,6 +83,17 @@ class VersionFactory(DjangoModelFactory):
RatingFactory, size=lambda: random.randint(1, 50), factory_related_name='version' RatingFactory, size=lambda: random.randint(1, 50), factory_related_name='version'
) )
@factory.post_generation
def platforms(self, create, extracted, **kwargs):
if not create:
return
if not extracted:
return
tags = Platform.objects.filter(slug__in=extracted)
self.platforms.add(*tags)
@factory.post_generation @factory.post_generation
def tags(self, create, extracted, **kwargs): def tags(self, create, extracted, **kwargs):
if not create: if not create:

View File

@ -151,6 +151,7 @@ class VersionAdmin(admin.ModelAdmin):
'date_modified', 'date_modified',
'licenses', 'licenses',
'tags', 'tags',
'platforms',
) )
search_fields = ( search_fields = (
'id', 'id',
@ -188,6 +189,7 @@ class VersionAdmin(admin.ModelAdmin):
'tags', 'tags',
'file', 'file',
'permissions', 'permissions',
'platforms',
), ),
}, },
), ),
@ -224,6 +226,10 @@ class LicenseAdmin(admin.ModelAdmin):
list_display = ('name', 'slug', 'url') list_display = ('name', 'slug', 'url')
class PlatformAdmin(admin.ModelAdmin):
list_display = ('name', 'slug')
class TagAdmin(admin.ModelAdmin): class TagAdmin(admin.ModelAdmin):
model = Tag model = Tag
list_display = ('name', 'slug', 'type') list_display = ('name', 'slug', 'type')
@ -244,5 +250,6 @@ admin.site.register(models.Extension, ExtensionAdmin)
admin.site.register(models.Version, VersionAdmin) admin.site.register(models.Version, VersionAdmin)
admin.site.register(models.Maintainer, MaintainerAdmin) admin.site.register(models.Maintainer, MaintainerAdmin)
admin.site.register(models.License, LicenseAdmin) admin.site.register(models.License, LicenseAdmin)
admin.site.register(models.Platform, PlatformAdmin)
admin.site.register(models.Tag, TagAdmin) admin.site.register(models.Tag, TagAdmin)
admin.site.register(models.VersionPermission, VersionPermissionAdmin) admin.site.register(models.VersionPermission, VersionPermissionAdmin)

View File

@ -0,0 +1,38 @@
# Generated by Django 4.2.11 on 2024-05-14 11:06
from django.db import migrations, models
def populate_platforms(apps, schema_editor):
Platform = apps.get_model('extensions', 'Platform')
for p in ["windows-amd64", "windows-arm64", "macos-x86_64", "macos-arm64", "linux-x86_64"]:
Platform(name=p, slug=p).save()
class Migration(migrations.Migration):
dependencies = [
('extensions', '0029_remove_extensionreviewerflags_extension_and_more'),
]
operations = [
migrations.CreateModel(
name='Platform',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('date_created', models.DateTimeField(auto_now_add=True)),
('date_modified', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=128, unique=True)),
('slug', models.SlugField(help_text='A platform tag, see https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/', unique=True)),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='version',
name='platforms',
field=models.ManyToManyField(blank=True, related_name='versions', to='extensions.platform'),
),
migrations.RunPython(populate_platforms),
]

View File

@ -99,6 +99,23 @@ class License(CreatedModifiedMixin, models.Model):
return cls.objects.filter(slug__startswith=slug).first() return cls.objects.filter(slug__startswith=slug).first()
class Platform(CreatedModifiedMixin, models.Model):
name = models.CharField(max_length=128, null=False, blank=False, unique=True)
slug = models.SlugField(
blank=False,
null=False,
help_text='A platform tag, see https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/', # noqa
unique=True,
)
def __str__(self) -> str:
return f'{self.name}'
@classmethod
def get_by_slug(cls, slug: str):
return cls.objects.filter(slug__startswith=slug).first()
class ExtensionManager(models.Manager): class ExtensionManager(models.Manager):
@property @property
def listed(self): def listed(self):
@ -441,8 +458,9 @@ class VersionManager(models.Manager):
def update_or_create(self, *args, **kwargs): def update_or_create(self, *args, **kwargs):
# Stash the ManyToMany to be created after the Version has a valid ID already # Stash the ManyToMany to be created after the Version has a valid ID already
permissions = kwargs.pop('permissions', [])
licenses = kwargs.pop('licenses', []) licenses = kwargs.pop('licenses', [])
permissions = kwargs.pop('permissions', [])
platforms = kwargs.pop('platforms', [])
tags = kwargs.pop('tags', []) tags = kwargs.pop('tags', [])
version, result = super().update_or_create(*args, **kwargs) version, result = super().update_or_create(*args, **kwargs)
@ -450,6 +468,7 @@ class VersionManager(models.Manager):
# Add the ManyToMany to the already initialized Version # Add the ManyToMany to the already initialized Version
version.set_initial_licenses(licenses) version.set_initial_licenses(licenses)
version.set_initial_permissions(permissions) version.set_initial_permissions(permissions)
version.set_initial_platforms(platforms)
version.set_initial_tags(tags) version.set_initial_tags(tags)
return version, result return version, result
@ -518,6 +537,7 @@ class Version(CreatedModifiedMixin, RatingMixin, TrackChangesMixin, models.Model
) )
permissions = models.ManyToManyField(VersionPermission, related_name='versions', blank=True) permissions = models.ManyToManyField(VersionPermission, related_name='versions', blank=True)
platforms = models.ManyToManyField(Platform, related_name='versions', blank=True)
release_notes = models.TextField(help_text=common.help_texts.markdown, blank=True) release_notes = models.TextField(help_text=common.help_texts.markdown, blank=True)
@ -546,6 +566,14 @@ class Version(CreatedModifiedMixin, RatingMixin, TrackChangesMixin, models.Model
permission = VersionPermission.get_by_slug(permission_name) permission = VersionPermission.get_by_slug(permission_name)
self.permissions.add(permission) self.permissions.add(permission)
def set_initial_platforms(self, _platforms):
if not _platforms:
return
for slug in _platforms:
platform = Platform.get_by_slug(slug)
self.platforms.add(platform)
def set_initial_licenses(self, _licenses): def set_initial_licenses(self, _licenses):
if not _licenses: if not _licenses:
return return

View File

@ -0,0 +1,10 @@
{% if version.platforms.all %}
<div>
Supported platforms:
<ul>
{% for p in version.platforms.all %}
<li>{{p.name}}</li>
{% endfor %}
</ul>
</div>
{% endif %}

View File

@ -180,7 +180,10 @@
<div class="dl-row"> <div class="dl-row">
<div class="dl-col"> <div class="dl-col">
<dt>{% trans 'Compatibility' %}</dt> <dt>{% trans 'Compatibility' %}</dt>
<dd>{% include "extensions/components/blender_version.html" with version=latest %}</dd> <dd>
{% include "extensions/components/blender_version.html" with version=latest %}
{% include "extensions/components/platforms.html" with version=latest %}
</dd>
</div> </div>
</div> </div>

View File

@ -56,7 +56,10 @@
<div class="dl-row"> <div class="dl-row">
<div class="dl-col"> <div class="dl-col">
<dt>{% trans 'Compatibility' %}</dt> <dt>{% trans 'Compatibility' %}</dt>
<dd>{% include "extensions/components/blender_version.html" with version=version %}</dd> <dd>
{% include "extensions/components/blender_version.html" with version=version %}
{% include "extensions/components/platforms.html" with version=version %}
</dd>
</div> </div>
</div> </div>
<div class="dl-row"> <div class="dl-row">

Binary file not shown.

View File

@ -84,6 +84,11 @@ EXPECTED_VALIDATION_ERRORS = {
}, },
'invalid-manifest-toml.zip': {'source': ['Could not parse the manifest file.']}, 'invalid-manifest-toml.zip': {'source': ['Could not parse the manifest file.']},
'invalid-theme-multiple-xmls.zip': {'source': ['A theme should have exactly one XML file.']}, 'invalid-theme-multiple-xmls.zip': {'source': ['A theme should have exactly one XML file.']},
'invalid-missing-wheels.zip': {
'source': [
'A declared wheel is missing in the zip file, expected path: addon/./wheels/test-wheel-whatever.whl'
]
},
} }
POST_DATA = { POST_DATA = {
'preview_set-TOTAL_FORMS': ['0'], 'preview_set-TOTAL_FORMS': ['0'],
@ -169,13 +174,13 @@ class SubmitFileTest(TestCase):
user = UserFactory() user = UserFactory()
self.client.force_login(user) self.client.force_login(user)
for test_archive, extected_errors in EXPECTED_VALIDATION_ERRORS.items(): for test_archive, expected_errors in EXPECTED_VALIDATION_ERRORS.items():
with self.subTest(test_archive=test_archive): with self.subTest(test_archive=test_archive):
with open(TEST_FILES_DIR / test_archive, 'rb') as fp: with open(TEST_FILES_DIR / test_archive, 'rb') as fp:
response = self.client.post(self.url, {'source': fp, 'agreed_with_terms': True}) response = self.client.post(self.url, {'source': fp, 'agreed_with_terms': True})
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertDictEqual(response.context['form'].errors, extected_errors) self.assertDictEqual(response.context['form'].errors, expected_errors)
def test_addon_without_top_level_directory(self): def test_addon_without_top_level_directory(self):
self.assertEqual(Extension.objects.count(), 0) self.assertEqual(Extension.objects.count(), 0)

View File

@ -112,6 +112,18 @@ class ApiViewsTest(_BaseTestCase):
).json() ).json()
self.assertEqual(len(json3['data']), 3) self.assertEqual(len(json3['data']), 3)
def test_platform_filter(self):
create_approved_version(platforms=['windows-amd64'])
create_approved_version(platforms=['windows-arm64'])
create_approved_version()
url = reverse('extensions:api')
json = self.client.get(
url + '?platform=windows-amd64',
HTTP_ACCEPT='application/json',
).json()
self.assertEqual(len(json['data']), 2)
def test_blender_version_filter_latest_not_max_version(self): def test_blender_version_filter_latest_not_max_version(self):
version = create_approved_version(blender_version_min='4.0.1') version = create_approved_version(blender_version_min='4.0.1')
version.date_created version.date_created

View File

@ -7,7 +7,7 @@ from drf_spectacular.utils import OpenApiParameter, extend_schema
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from common.compare import is_in_version_range, version from common.compare import is_in_version_range, version
from extensions.models import Extension from extensions.models import Extension, Platform
from extensions.utils import clean_json_dictionary_from_optional_fields from extensions.utils import clean_json_dictionary_from_optional_fields
@ -20,7 +20,10 @@ log = logging.getLogger(__name__)
class ListedExtensionsSerializer(serializers.ModelSerializer): class ListedExtensionsSerializer(serializers.ModelSerializer):
error_messages = { error_messages = {
"invalid_version": "Invalid version: use full semantic versioning like 4.2.0." "invalid_blender_version": "Invalid blender_version: use full semantic versioning like "
"4.2.0.",
"invalid_platform": "Invalid platform: use notation specified in "
"https://developer.blender.org/docs/features/extensions/schema/1.0.0/",
} }
class Meta: class Meta:
@ -30,16 +33,22 @@ class ListedExtensionsSerializer(serializers.ModelSerializer):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.request = kwargs.pop('request', None) self.request = kwargs.pop('request', None)
self.blender_version = kwargs.pop('blender_version', None) self.blender_version = kwargs.pop('blender_version', None)
self.platform = kwargs.pop('platform', None)
self._validate() self._validate()
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def _validate(self): def _validate(self):
if self.blender_version is None: if self.blender_version:
return try:
try: version(self.blender_version)
version(self.blender_version) except ValidationError:
except ValidationError: self.fail('invalid_blender_version')
self.fail('invalid_version') if self.platform:
# FIXME change to an in-memory lookup?
try:
Platform.objects.get(slug=self.platform)
except Platform.DoesNotExist:
self.fail('invalid_platform')
def to_representation(self, instance): def to_representation(self, instance):
matching_version = None matching_version = None
@ -52,18 +61,19 @@ class ListedExtensionsSerializer(serializers.ModelSerializer):
if not versions: if not versions:
return None return None
versions = sorted(versions, key=lambda v: v.date_created, reverse=True) versions = sorted(versions, key=lambda v: v.date_created, reverse=True)
if self.blender_version: for v in versions:
for v in versions: if self.blender_version and not is_in_version_range(
if is_in_version_range( self.blender_version,
self.blender_version, v.blender_version_min,
v.blender_version_min, v.blender_version_max,
v.blender_version_max, ):
): continue
matching_version = v platform_slugs = set(p.slug for p in v.platforms.all())
break # empty platforms field matches any platform filter
else: if self.platform and not (not platform_slugs or self.platform in platform_slugs):
# same as latest_version, but without triggering a new queryset continue
matching_version = versions[0] matching_version = v
break
if not matching_version: if not matching_version:
return None return None
@ -85,6 +95,7 @@ class ListedExtensionsSerializer(serializers.ModelSerializer):
'maintainer': str(instance.authors.all()[0]), 'maintainer': str(instance.authors.all()[0]),
'license': [license_iter.slug for license_iter in matching_version.licenses.all()], 'license': [license_iter.slug for license_iter in matching_version.licenses.all()],
'permissions': [permission.slug for permission in matching_version.permissions.all()], 'permissions': [permission.slug for permission in matching_version.permissions.all()],
'platforms': [platform.slug for platform in matching_version.platforms.all()],
# TODO: handle copyright # TODO: handle copyright
'tags': [str(tag) for tag in matching_version.tags.all()], 'tags': [str(tag) for tag in matching_version.tags.all()],
} }
@ -101,21 +112,32 @@ class ExtensionsAPIView(APIView):
name="blender_version", name="blender_version",
description=("Blender version to check for compatibility"), description=("Blender version to check for compatibility"),
type=str, type=str,
) ),
OpenApiParameter(
name="platform",
description=("Platform to check for compatibility"),
type=str,
),
] ]
) )
def get(self, request): def get(self, request):
blender_version = request.GET.get('blender_version') blender_version = request.GET.get('blender_version')
platform = request.GET.get('platform')
qs = Extension.objects.listed.prefetch_related( qs = Extension.objects.listed.prefetch_related(
'authors', 'authors',
'versions', 'versions',
'versions__file', 'versions__file',
'versions__licenses', 'versions__licenses',
'versions__permissions', 'versions__permissions',
'versions__platforms',
'versions__tags', 'versions__tags',
).all() ).all()
serializer = self.serializer_class( serializer = self.serializer_class(
qs, blender_version=blender_version, request=request, many=True qs,
blender_version=blender_version,
platform=platform,
request=request,
many=True,
) )
data = [e for e in serializer.data if e is not None] data = [e for e in serializer.data if e is not None]
return Response( return Response(

View File

@ -45,6 +45,7 @@ class ExtensionDetailView(ExtensionQuerysetMixin, DetailView):
'versions__file', 'versions__file',
'versions__file__validation', 'versions__file__validation',
'versions__permissions', 'versions__permissions',
'versions__platforms',
) )
def get_object(self, queryset=None): def get_object(self, queryset=None):

View File

@ -39,6 +39,7 @@ class FileForm(forms.ModelForm):
'missing_or_multiple_theme_xml': _('A theme should have exactly one XML file.'), 'missing_or_multiple_theme_xml': _('A theme should have exactly one XML file.'),
'invalid_zip_archive': msg_only_zip_files, 'invalid_zip_archive': msg_only_zip_files,
'missing_manifest_toml': _('The manifest file is missing.'), 'missing_manifest_toml': _('The manifest file is missing.'),
'missing_wheel': _('A declared wheel is missing in the zip file, expected path: %(path)s'),
} }
class Meta: class Meta:
@ -143,7 +144,15 @@ class FileForm(forms.ModelForm):
manifest, error_codes = utils.read_manifest_from_zip(file_path) manifest, error_codes = utils.read_manifest_from_zip(file_path)
for code in error_codes: for code in error_codes:
errors.append(forms.ValidationError(self.error_messages[code])) if isinstance(code, dict):
errors.append(
forms.ValidationError(
self.error_messages[code['code']],
params=code['params'],
)
)
else:
errors.append(forms.ValidationError(self.error_messages[code]))
if errors: if errors:
self.add_error('source', errors) self.add_error('source', errors)

View File

@ -108,6 +108,9 @@ def read_manifest_from_zip(archive_path):
""" """
manifest_name = 'blender_manifest.toml' manifest_name = 'blender_manifest.toml'
error_codes = [] error_codes = []
file_list = []
manifest_content = None
try: try:
with zipfile.ZipFile(archive_path) as myzip: with zipfile.ZipFile(archive_path) as myzip:
bad_file = myzip.testzip() bad_file = myzip.testzip()
@ -129,34 +132,45 @@ def read_manifest_from_zip(archive_path):
error_codes.append('invalid_manifest_path') error_codes.append('invalid_manifest_path')
return None, error_codes return None, error_codes
# Extract the file content
with myzip.open(manifest_filepath) as file_content: with myzip.open(manifest_filepath) as file_content:
toml_content = toml.loads(file_content.read().decode()) manifest_content = file_content.read().decode()
# If manifest was parsed successfully, do additional type-specific validation
type_slug = toml_content['type']
if type_slug == 'theme':
theme_xmls = filter_paths_by_ext(file_list, '.xml')
if len(list(theme_xmls)) != 1:
error_codes.append('missing_or_multiple_theme_xml')
elif type_slug == 'add-on':
# __init__.py is expected to be next to the manifest
expected_init_path = os.path.join(os.path.dirname(manifest_filepath), '__init__.py')
init_filepath = find_exact_path(file_list, expected_init_path)
if not init_filepath:
error_codes.append('invalid_missing_init')
return toml_content, error_codes
except toml.decoder.TomlDecodeError as e:
logger.error(f"Manifest Error: {e.msg}")
error_codes.append('invalid_manifest_toml')
except Exception as e: except Exception as e:
logger.error(f"Error extracting from archive: {e}") logger.error(f"Error extracting from archive: {e}")
error_codes.append('invalid_zip_archive') error_codes.append('invalid_zip_archive')
return None, error_codes
return None, error_codes try:
toml_content = toml.loads(manifest_content)
except toml.decoder.TomlDecodeError as e:
logger.error(f"Manifest Error: {e.msg}")
error_codes.append('invalid_manifest_toml')
return None, error_codes
# If manifest was parsed successfully, do additional type-specific validation
type_slug = toml_content['type']
if type_slug == 'theme':
theme_xmls = filter_paths_by_ext(file_list, '.xml')
if len(list(theme_xmls)) != 1:
error_codes.append('missing_or_multiple_theme_xml')
elif type_slug == 'add-on':
# __init__.py is expected to be next to the manifest
expected_init_path = os.path.join(os.path.dirname(manifest_filepath), '__init__.py')
init_filepath = find_exact_path(file_list, expected_init_path)
if not init_filepath:
error_codes.append('invalid_missing_init')
wheels = toml_content.get('wheels')
if wheels:
for wheel in wheels:
expected_wheel_path = os.path.join(os.path.dirname(manifest_filepath), wheel)
wheel_filepath = find_exact_path(file_list, expected_wheel_path)
if not wheel_filepath:
error_codes.append(
{'code': 'missing_wheel', 'params': {'path': expected_wheel_path}}
)
return toml_content, error_codes
def guess_mimetype_from_ext(file_name: str) -> str: def guess_mimetype_from_ext(file_name: str) -> str:

View File

@ -7,7 +7,13 @@ from django.utils.deconstruct import deconstructible
from django.utils.html import escape from django.utils.html import escape
from django.utils.safestring import mark_safe from django.utils.safestring import mark_safe
from extensions.models import Extension, License, VersionPermission, Tag from extensions.models import (
Extension,
License,
Platform,
Tag,
VersionPermission,
)
from constants.base import EXTENSION_TYPE_SLUGS_SINGULAR, EXTENSION_TYPE_CHOICES from constants.base import EXTENSION_TYPE_SLUGS_SINGULAR, EXTENSION_TYPE_CHOICES
from files.utils import guess_mimetype_from_ext, guess_mimetype_from_content from files.utils import guess_mimetype_from_ext, guess_mimetype_from_content
@ -361,6 +367,53 @@ class PermissionsValidator:
return mark_safe(error_message) return mark_safe(error_message)
class PlatformsValidator:
"""See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/"""
example = ["windows-amd64", "linux-x86_64"]
@classmethod
def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
"""Return error message if there is any license that is not accepted by the site"""
is_error = False
error_message = ""
unknown_value = None
if type(value) != list:
is_error = True
else:
for platform in value:
if Platform.get_by_slug(platform):
continue
is_error = True
unknown_value = platform
break
if not is_error:
return
error_message = mark_safe(
f'Manifest value error: <code>platforms</code> expects a list of '
f'supported platforms. e.g., {cls.example}.'
)
if unknown_value:
error_message += mark_safe(f' Unknown value: {escape(unknown_value)}.')
return error_message
class WheelsValidator:
example = ["./wheels/mywheel-v1.0.0-py3-none-any.whl"]
@classmethod
def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
if type(value) != list:
return mark_safe(
f'Manifest value error: <code>wheels</code> expects a list of '
f'wheel files . e.g., {cls.example}.'
)
class VersionValidator: class VersionValidator:
example = '1.0.0' example = '1.0.0'
@ -489,10 +542,12 @@ class ManifestValidator:
} }
optional_fields = { optional_fields = {
'blender_version_max': VersionMaxValidator, 'blender_version_max': VersionMaxValidator,
'website': StringValidator,
'copyright': ListValidator, 'copyright': ListValidator,
'permissions': PermissionsValidator, 'permissions': PermissionsValidator,
'platforms': PlatformsValidator,
'tags': TagsValidator, 'tags': TagsValidator,
'website': StringValidator,
'wheels': WheelsValidator,
} }
all_fields = {**mandatory_fields, **optional_fields} all_fields = {**mandatory_fields, **optional_fields}