common/management/commands/generate_fake_data.py

@@ -64,6 +64,7 @@ class Command(BaseCommand):
             file__status=File.STATUSES.APPROVED,
             # extension__status=Extension.STATUSES.APPROVED,
             extension__name='Blender Kitsu',
+            extension__extension_id='blender_kitsu',
             extension__tags=['Generic'],
             extension__description=EXAMPLE_DESCRIPTION,
             extension__doc_url='https://studio.blender.org/',

common/tests/factories/extensions.py

@@ -18,6 +18,7 @@ class ExtensionFactory(DjangoModelFactory):
         model = Extension
 
     name = factory.Faker('catch_phrase')
+    extension_id = factory.Faker('slug')
     description = factory.LazyAttribute(
         lambda _: fake_markdown.post(size=random.choice(('medium', 'large')))
     )
@@ -46,6 +47,9 @@ class ExtensionFactory(DjangoModelFactory):
             for _ in extracted:
                 _.extension_preview.create(caption='asdfasdfas asdfasd', extension=self)
 
+    @factory.post_generation
+    def process_extension_id(self, created, extracted, **kwargs):
+        self.extension_id = self.extension_id.replace("-", "_")
 
 class RatingFactory(DjangoModelFactory):
     class Meta:

extensions/migrations/0009_extension_extension_id.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.0.6 on 2024-01-18 15:37
+
+from django.db import migrations, models
+
+
+def set_initial_extension_id(apps, schema_editor):
+    model = apps.get_model('extensions', 'extension')
+    for ob in model.objects.all():
+        # For the records, this doesn't garantee that the slug matches the file,
+        # but it is overkill for now to do advanced migration.
+        # the idea is to flatten out the migrations soon after this.
+        ob.extension_id = ob.slug.replace("-", "_")
+        ob.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('extensions', '0008_alter_version_blender_version_max_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='extension',
+            name='extension_id',
+            field=models.CharField(max_length=255, default="", unique=False),
+            preserve_default=False,
+        ),
+        migrations.RunPython(set_initial_extension_id),
+        migrations.AlterField(
+            model_name='extension',
+            name='extension_id',
+            field=models.CharField(max_length=255, unique=True),
+        ),
+    ]

extensions/models.py

@@ -151,6 +151,7 @@ class Extension(
 
     type = models.PositiveSmallIntegerField(choices=TYPES, default=TYPES.BPY, editable=False)
     slug = models.SlugField(unique=True, null=False, blank=False, editable=False)
+    extension_id = models.CharField(max_length=255, unique=True, null=False, blank=False)
     name = models.CharField(max_length=255, unique=True, null=False, blank=False)
     description = models.TextField(help_text=common.help_texts.markdown)
     tagline = models.CharField(max_length=128, null=False, blank=False, help_text='A very short description')

extensions/tests/test_manifest.py

@@ -0,0 +1,176 @@
+from typing import List
+
+from django.test import TestCase
+from django.urls import reverse
+
+from common.tests.factories.extensions import create_approved_version
+from common.tests.factories.files import FileFactory
+from common.tests.factories.users import UserFactory
+from constants.licenses import LICENSE_GPL3
+from extensions.models import Extension
+from files.models import File
+
+import tempfile
+import shutil
+import os
+import zipfile
+import toml
+
+
+import tempfile
+
+
+META_DATA = {
+    "id": "an_id",
+    "name": "Theme",
+    "description": "A theme.",
+    "version": "0.1.0",
+    "type": "theme",
+    "license": LICENSE_GPL3.slug,
+    "blender_version_min": "2.9.3",
+    "blender_version_max": "4.0.0",
+    }
+
+
+class ValidateManifestTest(TestCase):
+    fixtures = ['licenses']
+
+    def setUp(self):
+        super().setUp()
+
+        self.user = UserFactory()
+        self.temp_directory = tempfile.mkdtemp()
+
+        file_data = {
+            "name": "Blender Kitsu",
+            "id": "blender_kitsu",
+            "version": "0.1.5",
+        }
+        self.file = self._create_file_from_data("blender_kitsu_1.5.0.zip", file_data, self.user)
+
+    def tearDown(self):
+        super().tearDown()
+        shutil.rmtree(self.temp_directory)
+
+    @classmethod
+    def _get_submit_url(cls):
+        return reverse('extensions:submit')
+
+    def _create_file_from_data(self, filename, file_data, user):
+        output_path = os.path.join(self.temp_directory, filename)
+        manifest_path = os.path.join(self.temp_directory, "manifest.toml")
+        combined_meta_data = META_DATA.copy()
+        combined_meta_data.update(file_data)
+
+        version = combined_meta_data.get("version", "0.1.0")
+        extension_id = combined_meta_data.get("id", "foobar").strip()
+
+        with open(manifest_path, "w") as manifest_file:
+            toml.dump(combined_meta_data, manifest_file)
+
+        with zipfile.ZipFile(output_path, "w") as my_zip:
+            arcname = f"{extension_id}-{version}/{os.path.basename(manifest_path)}"
+            my_zip.write(manifest_path, arcname=arcname)
+
+        os.remove(manifest_path)
+        return output_path
+
+    def test_validation_manifest_extension_id_hyphen(self):
+        self.assertEqual(Extension.objects.count(), 0)
+        user = UserFactory()
+        self.client.force_login(user)
+
+        file_data = {
+            "id": "id-with-hyphens",
+        }
+
+        bad_file = self._create_file_from_data("theme.zip", file_data, self.user)
+        with open (bad_file, 'rb') as fp:
+            response = self.client.post(self._get_submit_url(), {'source': fp, 'agreed_with_terms': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertDictEqual(
+            response.context['form'].errors,
+            {'source': ['Invalid id from extension manifest: "id-with-hyphens". No hyphens are allowed.']},
+        )
+
+    def test_validation_manifest_extension_id_spaces(self):
+        self.assertEqual(Extension.objects.count(), 0)
+        user = UserFactory()
+        self.client.force_login(user)
+
+        file_data = {
+            "id": "id with spaces",
+        }
+
+        bad_file = self._create_file_from_data("theme.zip", file_data, self.user)
+        with open (bad_file, 'rb') as fp:
+            response = self.client.post(self._get_submit_url(), {'source': fp, 'agreed_with_terms': True})
+
+
+        self.assertEqual(response.status_code, 200)
+        self.assertDictEqual(
+            response.context['form'].errors,
+            {'source': ['Invalid id from extension manifest: "id with spaces". Use a valid id consisting of Unicode letters, numbers or underscores.']},
+        )
+
+    def _create_valid_extension(self, extension_id):
+        return create_approved_version(
+            extension__name='Blender Kitsu',
+            extension__extension_id=extension_id,
+            version='0.1.5-alpha+f52258de',
+            file=FileFactory(
+                type=File.TYPES.THEME,
+                status=File.STATUSES.APPROVED,
+            ),
+        )
+
+    def test_validation_manifest_extension_id_clash(self):
+        """Test if we add two extensions with the same extension_id"""
+        self.assertEqual(Extension.objects.count(), 0)
+        self._create_valid_extension('blender_kitsu')
+        self.assertEqual(Extension.objects.count(), 1)
+
+        user = UserFactory()
+        self.client.force_login(user)
+
+        kitsu_1_5 = {
+            "name": "Blender Kitsu",
+            "id": "blender_kitsu",
+            "version": "0.1.5",
+        }
+
+        extension_file = self._create_file_from_data("theme.zip", kitsu_1_5, self.user)
+        with open (extension_file, 'rb') as fp:
+            response = self.client.post(self._get_submit_url(), {'source': fp, 'agreed_with_terms': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertDictEqual(
+            response.context['form'].errors,
+            {'source': ['The extension id in the manifest ("blender_kitsu") is already being used by another extension.']},
+        )
+
+    def test_validation_manifest_extension_id_mismatch(self):
+        """Test if we try to add a new version to an extension with a mismatched extension_id"""
+        self.assertEqual(Extension.objects.count(), 0)
+        version = self._create_valid_extension('blender_kitsu')
+        self.assertEqual(Extension.objects.count(), 1)
+
+        # The same author is to send a new version to thte same extension
+        self.client.force_login(version.file.user)
+
+        non_kitsu_1_6 = {
+            "name": "Blender Kitsu with a different ID",
+            "id": "non_kitsu",
+            "version": "0.1.6",
+        }
+
+        extension_file = self._create_file_from_data("theme.zip", non_kitsu_1_6, self.user)
+        with open (extension_file, 'rb') as fp:
+            response = self.client.post(version.extension.get_new_version_url(), {'source': fp, 'agreed_with_terms': True})
+
+        self.assertEqual(response.status_code, 200)
+        self.assertDictEqual(
+            response.context['form'].errors,
+            {'source': ['The extension id in the manifest ("non_kitsu") doesn\'t match the expected one for this extension ("blender_kitsu").']},
+        )

extensions/tests/test_submit.py

@@ -19,6 +19,7 @@ EXPECTED_EXTENSION_DATA = {
     'edit_breakdown-0.1.0.zip': {
         'metadata': {
             'description': 'Get insight on the complexity of an edit',
+            'id': 'edit_breakdown',
             'name': 'Edit Breakdown',
             'version': '0.1.0',
             'blender_version_min': '2.83.0',
@@ -33,6 +34,7 @@ EXPECTED_EXTENSION_DATA = {
         'metadata': {
             'description': 'Various tools for handle geodata',
             'name': 'BlenderGIS',
+            'id': 'blender_gis',
             'version': '2.2.8',
             'blender_version_min': '2.83.0',
             'type': 'add-on'
@@ -46,6 +48,7 @@ EXPECTED_EXTENSION_DATA = {
         'metadata': {
             'description': 'A collection of tools and settings to improve productivity',
             'name': 'Amaranth',
+            'id': 'amaranth',
             'version': '1.0.8',
             'blender_version_min': '2.83.0',
             'type': 'add-on'
@@ -469,7 +472,7 @@ class NewVersionTest(TestCase):
     fixtures = ['tags', 'licenses']
 
     def setUp(self):
-        self.version = create_version()
+        self.version = create_version(extension__extension_id='amaranth')
         self.extension = self.version.extension
         self.url = self.extension.get_new_version_url()
 

extensions/views/manage.py

@@ -186,6 +186,7 @@ class NewVersionView(
     def get_form_kwargs(self):
         kwargs = super().get_form_kwargs()
         kwargs['request'] = self.request
+        kwargs['extension'] = self.extension
         return kwargs
 
     def get_success_url(self):

extensions/views/public.py

@@ -96,7 +96,7 @@ class SearchView(ListedExtensionsView):
                 author=str(e.authors.first()),
             )
             # Keep the optional values outside the final json
-            extensions[e.slug] = (asdict(se, dict_factory=lambda x: {k: v for (k, v) in x if v is not None}))
+            extensions[e.extension_id] = (asdict(se, dict_factory=lambda x: {k: v for (k, v) in x if v is not None}))
         return extensions
 
     def _get_support_id_by_slug(self):

files/forms.py

@@ -6,7 +6,7 @@ import tempfile
 from django import forms
 from django.utils.safestring import mark_safe
 
-from .validators import CustomFileExtensionValidator
+from .validators import CustomFileExtensionValidator, ExtensionIDManifestValidator
 from constants.base import (
     EXTENSION_TYPE_SLUGS_SINGULAR,
     VALID_SOURCE_EXTENSIONS,
@@ -45,6 +45,7 @@ class FileForm(forms.ModelForm):
 
     def __init__(self, *args, **kwargs):
         self.request = kwargs.pop('request')
+        self.extension = kwargs.pop('extension', None)
         for field in self.base_fields:
             if field not in {'source', 'agreed_with_terms'}:
                 self.base_fields[field].required = False
@@ -118,6 +119,8 @@ class FileForm(forms.ModelForm):
 
             extension_types = {v: k for k, v in EXTENSION_TYPE_SLUGS_SINGULAR.items()}
 
+            ExtensionIDManifestValidator(manifest, self.extension)
+
             if errors:
                 raise forms.ValidationError({'source': errors}, code='invalid')
 

files/models.py

@@ -185,6 +185,7 @@ class File(CreatedModifiedMixin, TrackChangesMixin, SoftDeleteMixin, models.Mode
         """Return Extension-related data that was parsed from file's content."""
         data = self.metadata
 
+        extension_id = data.get('id')
         original_name = data.get('name', self.original_name)
         name_as_path = Path(original_name)
         for suffix in name_as_path.suffixes:
@@ -201,6 +202,7 @@ class File(CreatedModifiedMixin, TrackChangesMixin, SoftDeleteMixin, models.Mode
         return {
             'name': name,
             'slug': utils.slugify(name),
+            'extension_id': extension_id,
             'description': data.get('description'),
             'doc_url': data.get('doc_url', data.get('wiki_url')),
             'tracker_url': data.get('tracker_url'),

files/validators.py

@@ -1,8 +1,10 @@
 from pathlib import Path
 
+from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ValidationError
-from django.core.validators import FileExtensionValidator
+from django.core.validators import FileExtensionValidator, validate_unicode_slug
 
+from extensions.models import Extension
 
 class CustomFileExtensionValidator(FileExtensionValidator):
     """Allows extensions such as tar.gz."""
@@ -28,3 +30,69 @@ class CustomFileExtensionValidator(FileExtensionValidator):
                     "value": value,
                 },
             )
+
+
+class ExtensionIDManifestValidator:
+    """
+    Make sure the extension id is valid:
+      * Extension id consists of Unicode letters, numbers or underscores.
+      * Neither hyphens nor spaces are supported.
+      * Each extension id most be unique across all extensions.
+      * All versions of an extension must have the same extension id.
+    """
+    def __init__(self, manifest, extension_to_be_updated):
+        extension_id = manifest.get('id')
+
+        if extension_id is None:
+             raise ValidationError(
+                 {
+                    'source': [
+                        _('Missing field in manifest.toml: "id"'),
+                    ],
+                },
+                code='invalid',
+            )
+
+        if '-' in extension_id:
+            raise ValidationError(
+                {
+                    'source': [
+                        _(f'Invalid id from extension manifest: "{extension_id}". '
+                          'No hyphens are allowed.'),
+                    ],
+                },
+                code='invalid',
+            )
+
+        try:
+            validate_unicode_slug(extension_id)
+        except ValidationError:
+            raise ValidationError(
+                    {
+                        'source': [
+                            _(f'Invalid id from extension manifest: "{extension_id}". '
+                            'Use a valid id consisting of Unicode letters, numbers or underscores.'),
+                        ],
+                    },
+                    code='invalid',
+                )
+
+        if extension_to_be_updated is None:
+            if Extension.objects.filter(extension_id=extension_id).exists():
+                raise ValidationError(
+                    {
+                        'source': [
+                            _(f'The extension id in the manifest ("{extension_id}") is already being used by another extension.'),
+                        ],
+                    },
+                    code='invalid',
+                )
+        elif extension_to_be_updated.extension_id != extension_id:
+            raise ValidationError(
+                {
+                    'source': [
+                        _(f'The extension id in the manifest ("{extension_id}") doesn\'t match the expected one for this extension ("{extension_to_be_updated.extension_id}").'),
+                    ],
+                },
+                code='invalid',
+            )