extensions-website/files/validators.py

from semantic_version import Version
import logging
import re

from django.core.exceptions import ValidationError
from django.core.validators import URLValidator, validate_unicode_slug
from django.utils.deconstruct import deconstructible
from django.utils.html import escape
from django.utils.safestring import mark_safe

from extensions.models import (
    Extension,
    License,
    Platform,
    Tag,
    VersionPermission,
)
from constants.base import EXTENSION_TYPE_SLUGS_SINGULAR, EXTENSION_TYPE_CHOICES
from files.utils import guess_mimetype_from_ext, guess_mimetype_from_content
import files.models

logger = logging.getLogger(__name__)

# Define the regex pattern for major.minor.patch (do not include any additional label).
# https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
version_regex_pattern = re.compile(r'^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$')


@deconstructible
class FileMIMETypeValidator:
    code = 'invalid_mimetype'

    def __init__(self, allowed_mimetypes, message, code=None):
        self.allowed_mimetypes = {_type.lower() for _type in allowed_mimetypes}
        if message is not None:
            self.message = message
        if code is not None:
            self.code = code

    def __call__(self, value):
        # Guess MIME-type based on extension first
        mimetype_from_ext = guess_mimetype_from_ext(value.name)
        if mimetype_from_ext not in self.allowed_mimetypes:
            raise ValidationError(self.message, code=self.code)
        # Guess MIME-type based on file's content
        mimetype_from_bytes = guess_mimetype_from_content(value)
        if mimetype_from_bytes not in self.allowed_mimetypes:
            raise ValidationError(self.message, code=self.code)
        if mimetype_from_ext != mimetype_from_bytes:
            # This shouldn't happen, but libmagic's and mimetypes' mappings
            # might differ from distro to distro.
            logger.exception(
                "MIME-type from extension (%s) doesn't match content (%s)",
                mimetype_from_ext,
                mimetype_from_bytes,
            )
            raise ValidationError(self.message, code=self.code)

    def __eq__(self, other):
        return (
            isinstance(other, self.__class__)
            and self.allowed_mimetypes == other.allowed_mimetypes
            and self.message == other.message
            and self.code == other.code
        )


class ExtensionIDManifestValidator:
    """Make sure the extension id is valid:
    * Extension id consists of Unicode letters, numbers or underscores.
    * Neither hyphens nor spaces are supported.
    * Each extension id most be unique across all extensions.
    * All versions of an extension must have the same extension id.
    """

    def __init__(self, manifest, extension_to_be_updated):
        extension_id = manifest.get('id')

        if extension_id is None:
            raise ValidationError(
                {
                    'source': [
                        mark_safe('Missing field in blender_manifest.toml: <code>id</code>'),
                    ],
                },
                code='invalid',
            )

        if '-' in extension_id:
            raise ValidationError(
                {
                    'source': [
                        mark_safe(
                            "Invalid <code>id</code> from extension manifest: "
                            f'"{escape(extension_id)}". No hyphens are allowed.'
                        ),
                    ],
                },
                code='invalid',
            )

        try:
            validate_unicode_slug(extension_id)
        except ValidationError:
            raise ValidationError(
                {
                    'source': [
                        mark_safe(
                            f'Invalid <code>id</code> from extension manifest: '
                            f'"{escape(extension_id)}". '
                            f'Use a valid id consisting of Unicode letters, numbers or underscores.'
                        ),
                    ],
                },
                code='invalid',
            )

        if extension_to_be_updated is None:
            if Extension.objects.filter(extension_id=extension_id).exists():
                raise ValidationError(
                    {
                        'source': [
                            mark_safe(
                                f'The extension <code>id</code> in the manifest '
                                f'("{escape(extension_id)}") '
                                f'is already being used by another extension.'
                            ),
                        ],
                    },
                    code='invalid',
                )
        elif extension_to_be_updated.extension_id != extension_id:
            raise ValidationError(
                {
                    'source': [
                        mark_safe(
                            f'The extension <code>id</code> in the manifest '
                            f'("{escape(extension_id)}") doesn\'t match the expected one for'
                            f'this extension ("{escape(extension_to_be_updated.extension_id)}").'
                        ),
                    ],
                },
                code='invalid',
            )


class ExtensionNameManifestValidator:
    """Validates name uniqueness across extensions."""

    def __init__(self, manifest, extension_to_be_updated):
        name = manifest.get('name')

        extension = Extension.objects.filter(name=name).first()
        if extension:
            if (
                extension_to_be_updated is None
                or extension_to_be_updated.extension_id != extension.extension_id
            ):
                raise ValidationError(
                    {
                        'source': [
                            mark_safe(
                                f'The extension <code>name</code> in the manifest '
                                f'("{escape(name)}") '
                                f'is already being used by another extension.'
                            ),
                        ],
                    },
                    code='invalid',
                )


class ExtensionVersionManifestValidator:
    """Validates version existence in db and available platforms."""

    def __init__(self, manifest, extension_to_be_updated):

        # If the extension wasn't created yet, any version is valid
        if not extension_to_be_updated:
            return

        manifest_version = manifest.get('version')
        version = extension_to_be_updated.versions.filter(version=manifest_version).first()
        if not version:
            return
        # for existing versions only submissions for remaining platfroms are valid
        remaining_platforms = version.get_remaining_platforms()
        if platforms := files.models.File.parse_platforms_from_manifest(manifest):
            if diff := set(platforms) - remaining_platforms:
                raise ValidationError(
                    {
                        'source': [f'{version} already has files for {", ".join(diff)}'],
                    },
                    code='invalid',
                )
        else:
            if remaining_platforms:
                raise ValidationError(
                    {
                        'source': [
                            f'File upload for {version} is allowed only for remaining platforms: '
                            f'{", ".join(remaining_platforms)}'
                        ],
                    },
                    code='invalid',
                )
            else:
                raise ValidationError(
                    {
                        'source': [
                            f'The version {escape(manifest_version)} was already uploaded for this '
                            f'extension ({extension_to_be_updated.name})'
                        ],
                    },
                    code='invalid',
                )


class ManifestFieldValidator:
    @classmethod
    def validate(cls, *, name: str, value: object, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        assert not "ManifestFieldValidator must be inherited not to be used directly."


class SimpleValidator(ManifestFieldValidator):
    @classmethod
    def validate(cls, *, name: str, value: object, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        if not hasattr(cls, '_type') or not hasattr(cls, '_type_name'):
            assert not "SimpleValidator must be inherited not be used directly."
        if type(value) != cls._type:
            return mark_safe(
                f'Manifest value error: <code>{name}</code> should be of type: {cls._type_name}'
            )


class StringValidator(SimpleValidator):
    _type = str
    _type_name = 'Text: ""'
    example = 'text'


class ListValidator(SimpleValidator):
    _type = list
    _type_name = 'List of values []'
    example = ['string', 'another_string']


class LicenseValidator(ListValidator):
    example = ['SPDX:GPL-3.0-or-later']

    @classmethod
    def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
        """Return error message if there is any license that is not accepted by the site"""
        is_error = False
        error_message = ""
        gnu_gpl3_slug = "SPDX:GPL-3.0-or-later"

        if type(value) != list:
            is_error = True

        unknown_value = None
        for license in value:
            if License.get_by_slug(license):
                continue
            is_error = True
            unknown_value = license
            break

        if (
            type(value) is list
            and (manifest.get("type") == EXTENSION_TYPE_SLUGS_SINGULAR[EXTENSION_TYPE_CHOICES.BPY])
            and gnu_gpl3_slug not in value
        ):
            return mark_safe(
                f'Manifest value error: <code>license</code> for add-ons must be '
                f'<a href="https://spdx.org/licenses/GPL-3.0-or-later.html">GPL v3.0 or later</a>. '
                f'Additional license are possible, read the '
                f'<a href="https://docs.blender.org/manual/en/latest/'
                f'advanced/extensions/licenses.html">'
                f'documentation</a>. e.g., {cls.example}.'
            )

        if not is_error:
            return

        error_message = mark_safe(
            f'Manifest value error: <code>license</code> expects a list of '
            f'<a href="https://docs.blender.org/manual/en/dev/advanced/extensions/licenses.html">'
            f'supported licenses</a>. e.g., {cls.example}.'
        )
        if unknown_value:
            error_message += mark_safe(f' Unknown value: {escape(unknown_value)}.')

        return error_message


class TagsValidatorBase:
    @classmethod
    def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
        """Return error message if there is no tag, or if tag is not a valid one"""
        is_error = False
        type_name = EXTENSION_TYPE_SLUGS_SINGULAR[cls.type]

        unknown_value = None
        if type(value) != list:
            is_error = True
        else:
            for tag in value:
                if Tag.objects.filter(name=tag, type=cls.type):
                    continue
                is_error = True
                unknown_value = tag
                type_slug = manifest.get('type')
                logger.info(f'Tag unavailable for {type_slug}: {tag}')

        if not is_error:
            return

        error_message = mark_safe(
            f'Manifest value error: <code>tags</code> expects a list of '
            f'<a href="https://docs.blender.org/manual/en/dev/advanced/extensions/tags.html" '
            f'target="_blank"> supported {type_name} tags</a>. e.g., {cls.example}. '
        )
        if unknown_value:
            error_message += mark_safe(f' Unknown value: {escape(unknown_value)}.')
        return error_message


class TagsAddonsValidator(TagsValidatorBase):
    example = ['Animation', 'Sequencer']
    type = EXTENSION_TYPE_CHOICES.BPY


class TagsThemesValidator(TagsValidatorBase):
    example = ['Dark', 'Accessibility']
    type = EXTENSION_TYPE_CHOICES.THEME


class TagsValidator:
    example = ['User Interface']

    @classmethod
    def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
        """Return error message if there is no tag, or if tag is not a valid one"""
        tags_lookup = {
            EXTENSION_TYPE_SLUGS_SINGULAR[EXTENSION_TYPE_CHOICES.BPY]: TagsAddonsValidator,
            EXTENSION_TYPE_SLUGS_SINGULAR[EXTENSION_TYPE_CHOICES.THEME]: TagsThemesValidator,
        }

        type_slug = manifest.get('type')
        meta_class = tags_lookup.get(type_slug)

        if meta_class is None:
            return

        return meta_class.validate(name=name, value=value, manifest=manifest)


class TypeValidator:
    example = 'add-on'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if doesn´t contain one of the valid types."""
        is_error = False

        if (type(value) != str) or (not value):
            is_error = True
        elif value not in EXTENSION_TYPE_SLUGS_SINGULAR.values():
            is_error = True
            logger.info(f'Type unavailable: {value}')

        if not is_error:
            return

        error_message = (
            f'Manifest value error: <code>{name}</code> expects one of the supported types. '
            f'The supported types are: '
        )

        for _type in EXTENSION_TYPE_SLUGS_SINGULAR.values():
            error_message += f'"{_type}", '
        error_message = error_message[:-2] + '.'

        return mark_safe(error_message)


class PermissionsValidator:
    example = {
        'files': "Import/export FBX from/to disk",
        'network': "Need to sync motion-capture data to server",
    }

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if doesn´t contain a valid list of permissions."""
        is_error = False

        extension_type = manifest.get('type')
        is_theme = extension_type == EXTENSION_TYPE_SLUGS_SINGULAR[EXTENSION_TYPE_CHOICES.THEME]
        is_bpy = extension_type == EXTENSION_TYPE_SLUGS_SINGULAR[EXTENSION_TYPE_CHOICES.BPY]

        if is_theme:
            error_message = mark_safe(
                'Manifest value error: <code>permissions</code> not required for themes.'
            )
            return error_message

        # Let the wrong type error be handled elsewhere.
        if not is_bpy:
            return

        if not isinstance(value, dict) or not value:
            is_error = True
        else:
            for permission, reason in value.items():
                try:
                    VersionPermission.get_by_slug(permission)
                except VersionPermission.DoesNotExist:
                    is_error = True
                    logger.info(f'Permission unavailable: {permission}')
                if not isinstance(reason, str):
                    return mark_safe(
                        'Manifest value error: <code>permissions</code> reasons must be strings.'
                    )

                if not reason:
                    return mark_safe(
                        f'Manifest value error: <code>permissions</code> {permission} reason '
                        'is empty.'
                    )

                if reason[-1] in {'.', '!', '?'}:
                    return mark_safe(
                        'Manifest value error: <code>permissions</code> {permission} reason '
                        'cannot end with any punctuation (.!?).'
                    )

                max_length = 64
                if len(reason) > max_length:
                    return mark_safe(
                        f'Manifest value error: <code>permissions</code> {permission} reason is '
                        'too long ({len(value)}), '
                        f'max-length: {max_length} characters'
                    )

        if not is_error:
            return

        error_message = (
            'Manifest value error: <code>permissions</code> expects key/value pairs of '
            'supported permissions with the reasons why they are required. '
            'The supported permissions are: '
        )

        for permission_ob in VersionPermission.objects.all():
            error_message += f'{permission_ob.slug}, '
        error_message = error_message[:-2] + '.'

        return mark_safe(error_message)


class PlatformsValidator:
    """See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/"""

    example = ["windows-x64", "linux-x64"]

    @classmethod
    def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
        """Return error message if there is any license that is not accepted by the site"""
        is_error = False
        error_message = ""

        unknown_value = None
        if type(value) != list:
            is_error = True
        else:
            for platform in value:
                if Platform.get_by_slug(platform):
                    continue
                is_error = True
                unknown_value = platform
                break

        if not is_error:
            return

        error_message = mark_safe(
            f'Manifest value error: <code>platforms</code> expects a list of '
            f'supported platforms. e.g., {cls.example}.'
        )
        if unknown_value:
            error_message += mark_safe(f' Unknown value: {escape(unknown_value)}.')

        return error_message


class BuildValidator:
    example = {
        "generated": {
            "platforms": ["linux-x64"],
            "wheels": ["./wheels/mywheel-v1.0.0-py3-none-any.whl"],
        }
    }

    @classmethod
    def validate(cls, *, name: str, value: dict, manifest: dict) -> str:
        if 'generated' not in value:
            return
        if 'platforms' in value['generated']:
            if plaforms_error := PlatformsValidator.validate(
                name=name,
                value=value['generated']['platforms'],
                manifest=manifest,
            ):
                return plaforms_error

        if 'wheels' in value['generated']:
            if wheels_error := WheelsValidator.validate(
                name=name,
                value=value['generated']['wheels'],
                manifest=manifest,
            ):
                return wheels_error


class WheelsValidator:
    example = ["./wheels/mywheel-v1.0.0-py3-none-any.whl"]

    @classmethod
    def validate(cls, *, name: str, value: list[str], manifest: dict) -> str:
        if type(value) != list:
            return mark_safe(
                f'Manifest value error: <code>wheels</code> expects a list of '
                f'wheel files . e.g., {cls.example}.'
            )


class VersionValidator:
    example = '1.0.0'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        try:
            Version(value)
        except (ValueError, TypeError):
            # ValueError happens when passing an invalid version, like "2.9"
            # TypeError happens when e.g., passing an integer
            return mark_safe(
                f'Manifest value error: <code>{name}</code> should follow a '
                f'<a href="https://semver.org/" target="_blank">semantic version</a>. '
                f'e.g., "{cls.example}"'
            )


class SemVerNoLabelValidator(StringValidator):
    """These version are a sub-set of the semantic version

    They  only support major.minor.patch.
    """

    @staticmethod
    def is_version_string(version):
        return version_regex_pattern.match(version)

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        err_message = mark_safe(
            f'Manifest value error: <code>{name}</code> should follow a '
            f'<a href="https://semver.org/" target="_blank">semantic version</a> '
            f'with only major.minor.patch '
            f'e.g., "{cls.example}"'
        )

        if super().validate(name=name, value=value, manifest=manifest):
            return err_message

        # Make sure we only have major.minor.patch.
        if not cls.is_version_string(version=value):
            return err_message


class SchemaVersionValidator(SemVerNoLabelValidator):
    example = '1.0.0'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        if err_message := super().validate(name=name, value=value, manifest=manifest):
            return err_message

        valid_schemas = {
            "1.0.0",
        }

        if value not in valid_schemas:
            # TODO make a user manual page with the list of the different schemas.
            return mark_safe(
                f'Manifest value error: <code>schema</code> version ({escape(value)}) '
                f'<a href="https://docs.blender.org/manual/en/dev/advanced/extensions/'
                f'getting_started.html#manifest" target="_blank">not supported</a>.'
            )


class VersionMinValidator(SemVerNoLabelValidator):
    example = '4.2.0'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        if err_message := super().validate(name=name, value=value, manifest=manifest):
            return err_message

        # Extensions were created in 4.2.0
        if Version(value) < Version('4.2.0'):
            return mark_safe(
                'Manifest value error: <code>blender_version_min</code> should be at least "4.2.0"'
            )


class VersionMaxValidator(SemVerNoLabelValidator):
    example = '4.3.0'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if cannot validate, otherwise returns nothing"""
        if err_message := super().validate(name=name, value=value, manifest=manifest):
            return err_message

        min = None
        max = None
        try:
            min = Version(manifest.get('blender_version_min'))
        except (ValueError, TypeError):
            # assuming that VersionMinValidator has caught this and reported properly
            return

        max = Version(value)
        if max <= min:
            return mark_safe(
                'Manifest value error: <code>blender_version_max</code> should be greater than '
                '<code>blender_version_min</code>'
            )


class TaglineValidator(StringValidator):
    example = 'Short description of my extension'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        """Return error message if has period at the end of the line or is too long."""
        if err_message := super().validate(name=name, value=value, manifest=manifest):
            return err_message

        if not value:
            return mark_safe('Manifest value error: <code>tagline</code> cannot be empty.')

        if value[-1] in {'.', '!', '?'}:
            return mark_safe(
                "Manifest value error: <code>tagline</code> cannot end with any punctuation (.!?)."
            )

        # TODO: get this from the model, the following code was supposed to work, however
        # it does not (it did for Extensions).
        # Version._meta.get_field('tagline').max_length
        max_length = 64
        if len(value) > max_length:
            return mark_safe(
                f'Manifest value error: <code>tagline</code> is too long ({len(value)}), '
                f'max-length: {max_length} characters'
            )


class WebsiteValidator(StringValidator):
    example = 'https://extensions.blender.org/'

    @classmethod
    def validate(cls, *, name: str, value: str, manifest: dict) -> str:
        if not value:
            return
        if len(value) > 200:
            return mark_safe(
                'Manifest value error: <code>website</code> must not exceed 200 characters',
            )
        try:
            URLValidator()(value)
        except ValidationError:
            return mark_safe(
                'Manifest value error: <code>website</code> must be a valid URL',
            )


class ManifestValidator:
    """Make sure the manifest has all the expected fields."""

    mandatory_fields = {
        'blender_version_min': VersionMinValidator,
        'id': StringValidator,
        'license': LicenseValidator,
        'maintainer': StringValidator,
        'name': StringValidator,
        'schema_version': SchemaVersionValidator,
        'tagline': TaglineValidator,
        'type': TypeValidator,
        'version': VersionValidator,
    }
    optional_fields = {
        'blender_version_max': VersionMaxValidator,
        'build': BuildValidator,
        'copyright': ListValidator,
        'permissions': PermissionsValidator,
        'platforms': PlatformsValidator,
        'tags': TagsValidator,
        'website': WebsiteValidator,
        'wheels': WheelsValidator,
    }
    all_fields = {**mandatory_fields, **optional_fields}

    def __init__(self, manifest):
        missing_fields = []
        wrong_fields = []

        for field_name, field_validator in self.mandatory_fields.items():
            field_value = manifest.get(field_name)
            if field_value is None:
                missing_fields.append(field_name)
            elif err_message := field_validator.validate(
                name=field_name, value=field_value, manifest=manifest
            ):
                wrong_fields.append(err_message)

        for field_name, field_validator in self.optional_fields.items():
            field_value = manifest.get(field_name)
            if field_value is None:
                continue
            elif err_message := field_validator.validate(
                name=field_name, value=field_value, manifest=manifest
            ):
                wrong_fields.append(err_message)

        if not (missing_fields or wrong_fields):
            return

        errors = []

        if missing_fields:
            errors.append(
                'The following values are missing from the manifest file: '
                f'{", ".join(missing_fields)}'
            )

        # Add the wrong field error messages to the general errors.
        errors += wrong_fields

        raise ValidationError(
            {
                'source': errors,
            },
            code='invalid',
        )