Merge branch 'master' into tmp-buildbot-cleanup

Add snap configuration file used by Buildbot snap store steps1

build_files/buildbot/README.md

@@ -1,70 +1,4 @@
-Blender Buildbot
-================
+Buildbot Configuration
+=====================
 
-Code signing
-------------
-
-Code signing is done as part of INSTALL target, which makes it possible to sign
-files which are aimed into a bundle and coming from a non-signed source (such as
-libraries SVN).
-
-This is achieved by specifying `worker_codesign.cmake` as a post-install script
-run by CMake. This CMake script simply involves an utility script written in
-Python which takes care of an actual signing.
-
-### Configuration
-
-Client configuration doesn't need anything special, other than variable
-`SHARED_STORAGE_DIR` pointing to a location which is watched by a server.
-This is done in `config_builder.py` file and is stored in Git (which makes it
-possible to have almost zero-configuration buildbot machines).
-
-Server configuration requires copying `config_server_template.py` under the
-name of `config_server.py` and tweaking values, which are platform-specific.
-
-#### Windows configuration
-
-There are two things which are needed on Windows in order to have code signing
-to work:
-
-- `TIMESTAMP_AUTHORITY_URL` which is most likely set http://timestamp.digicert.com
-- `CERTIFICATE_FILEPATH` which is a full file path to a PKCS #12 key (.pfx).
-
-## Tips
-
-### Self-signed certificate on Windows
-
-It is easiest to test configuration using self-signed certificate.
-
-The certificate manipulation utilities are coming with Windows SDK.
-Unfortunately, they are not added to PATH. Here is an example of how to make
-sure they are easily available:
-
-```
-set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH%
-set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64;%PATH%
-```
-
-Generate CA:
-
-```
-makecert -r -pe -n "CN=Blender Test CA" -ss CA -sr CurrentUser -a sha256 ^
-         -cy authority -sky signature -sv BlenderTestCA.pvk BlenderTestCA.cer
-```
-
-Import the generated CA:
-
-```
-certutil -user -addstore Root BlenderTestCA.cer
-```
-
-Create self-signed certificate and pack it into PKCS #12:
-
-```
-makecert -pe -n "CN=Blender Test SPC" -a sha256 -cy end ^
-         -sky signature ^
-         -ic BlenderTestCA.cer -iv BlenderTestCA.pvk ^
-         -sv BlenderTestSPC.pvk BlenderTestSPC.cer
-
-pvk2pfx -pvk BlenderTestSPC.pvk -spc BlenderTestSPC.cer -pfx BlenderTestSPC.pfx
-```
+Files used by Buildbot's `compile-code` step.

build_files/buildbot/buildbot_utils.py

@@ -1,127 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import argparse
-import os
-import re
-import subprocess
-import sys
-
-
-def is_tool(name):
-    """Check whether `name` is on PATH and marked as executable."""
-
-    # from whichcraft import which
-    from shutil import which
-
-    return which(name) is not None
-
-
-class Builder:
-    def __init__(self, name, branch, codesign):
-        self.name = name
-        self.branch = branch
-        self.is_release_branch = re.match("^blender-v(.*)-release$", branch) is not None
-        self.codesign = codesign
-
-        # Buildbot runs from build/ directory
-        self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git'))
-        self.build_dir = os.path.abspath(os.path.join('..', 'build'))
-        self.install_dir = os.path.abspath(os.path.join('..', 'install'))
-        self.upload_dir = os.path.abspath(os.path.join('..', 'install'))
-
-        # Detect platform
-        if name.startswith('mac'):
-            self.platform = 'mac'
-            self.command_prefix = []
-        elif name.startswith('linux'):
-            self.platform = 'linux'
-            if is_tool('scl'):
-                self.command_prefix = ['scl', 'enable', 'devtoolset-9', '--']
-            else:
-                self.command_prefix = []
-        elif name.startswith('win'):
-            self.platform = 'win'
-            self.command_prefix = []
-        else:
-            raise ValueError('Unkonw platform for builder ' + self.platform)
-
-        # Always 64 bit now
-        self.bits = 64
-
-
-def create_builder_from_arguments():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('builder_name')
-    parser.add_argument('branch', default='master', nargs='?')
-    parser.add_argument("--codesign", action="store_true")
-    args = parser.parse_args()
-    return Builder(args.builder_name, args.branch, args.codesign)
-
-
-class VersionInfo:
-    def __init__(self, builder):
-        # Get version information
-        buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h")
-        blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
-
-        version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION'))
-        version_number_patch = int(self._parse_header_file(blender_h, 'BLENDER_VERSION_PATCH'))
-        version_numbers = (version_number // 100, version_number % 100, version_number_patch)
-        self.short_version = "%d.%d" % (version_numbers[0], version_numbers[1])
-        self.version = "%d.%d.%d" % version_numbers
-        self.version_cycle = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE')
-        self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
-
-        if self.version_cycle == "release":
-            # Final release
-            self.full_version = self.version
-            self.is_development_build = False
-        elif self.version_cycle == "rc":
-            # Release candidate
-            self.full_version = self.version + self.version_cycle
-            self.is_development_build = False
-        else:
-            # Development build
-            self.full_version = self.version + '-' + self.hash
-            self.is_development_build = True
-
-    def _parse_header_file(self, filename, define):
-        import re
-        regex = re.compile(r"^#\s*define\s+%s\s+(.*)" % define)
-        with open(filename, "r") as file:
-            for l in file:
-                match = regex.match(l)
-                if match:
-                    return match.group(1)
-        return None
-
-
-def call(cmd, env=None, exit_on_error=True):
-    print(' '.join(cmd))
-
-    # Flush to ensure correct order output on Windows.
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-    retcode = subprocess.call(cmd, env=env)
-    if exit_on_error and retcode != 0:
-        sys.exit(retcode)
-    return retcode

build_files/buildbot/codesign/absolute_and_relative_filename.py

@@ -1,81 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import List
-
-
-@dataclass
-class AbsoluteAndRelativeFileName:
-    """
-    Helper class which keeps track of absolute file path for a direct access and
-    corresponding relative path against given base.
-
-    The relative part is used to construct a file name within an archive which
-    contains files which are to be signed or which has been signed already
-    (depending on whether the archive is addressed to signing server or back
-    to the buildbot worker).
-    """
-
-    # Base directory which is where relative_filepath is relative to.
-    base_dir: Path
-
-    # Full absolute path of the corresponding file.
-    absolute_filepath: Path
-
-    # Derived from full file path, contains part of the path which is relative
-    # to a desired base path.
-    relative_filepath: Path
-
-    def __init__(self, base_dir: Path, filepath: Path):
-        self.base_dir = base_dir
-        self.absolute_filepath = filepath.resolve()
-        self.relative_filepath = self.absolute_filepath.relative_to(
-            self.base_dir)
-
-    @classmethod
-    def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName':
-        assert path.is_absolute()
-        assert path.is_file()
-
-        base_dir = path.parent
-        return AbsoluteAndRelativeFileName(base_dir, path)
-
-    @classmethod
-    def recursively_from_directory(cls, base_dir: Path) \
-            -> List['AbsoluteAndRelativeFileName']:
-        """
-        Create list of AbsoluteAndRelativeFileName for all the files in the
-        given directory.
-
-        NOTE: Result will be pointing to a resolved paths.
-        """
-        assert base_dir.is_absolute()
-        assert base_dir.is_dir()
-
-        base_dir = base_dir.resolve()
-
-        result = []
-        for filename in base_dir.glob('**/*'):
-            if not filename.is_file():
-                continue
-            result.append(AbsoluteAndRelativeFileName(base_dir, filename))
-        return result

build_files/buildbot/codesign/archive_with_indicator.py

@@ -1,245 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import dataclasses
-import json
-import os
-
-from pathlib import Path
-from typing import Optional
-
-import codesign.util as util
-
-
-class ArchiveStateError(Exception):
-    message: str
-
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)
-
-
-@dataclasses.dataclass
-class ArchiveState:
-    """
-    Additional information (state) of the archive
-
-    Includes information like expected file size of the archive file in the case
-    the archive file is expected to be successfully created.
-
-    If the archive can not be created, this state will contain error message
-    indicating details of error.
-    """
-
-    # Size in bytes of the corresponding archive.
-    file_size: Optional[int] = None
-
-    # Non-empty value indicates that error has happenned.
-    error_message: str = ''
-
-    def has_error(self) -> bool:
-        """
-        Check whether the archive is at error state
-        """
-
-        return self.error_message
-
-    def serialize_to_string(self) -> str:
-        payload = dataclasses.asdict(self)
-        return json.dumps(payload, sort_keys=True, indent=4)
-
-    def serialize_to_file(self, filepath: Path) -> None:
-        string = self.serialize_to_string()
-        filepath.write_text(string)
-
-    @classmethod
-    def deserialize_from_string(cls, string: str) -> 'ArchiveState':
-        try:
-            object_as_dict = json.loads(string)
-        except json.decoder.JSONDecodeError:
-            raise ArchiveStateError('Error parsing JSON')
-
-        return cls(**object_as_dict)
-
-    @classmethod
-    def deserialize_from_file(cls, filepath: Path):
-        string = filepath.read_text()
-        return cls.deserialize_from_string(string)
-
-
-class ArchiveWithIndicator:
-    """
-    The idea of this class is to wrap around logic which takes care of keeping
-    track of a name of an archive and synchronization routines between buildbot
-    worker and signing server.
-
-    The synchronization is done based on creating a special file after the
-    archive file is knowingly ready for access.
-    """
-
-    # Base directory where the archive is stored (basically, a basename() of
-    # the absolute archive file name).
-    #
-    # For example, 'X:\\TEMP\\'.
-    base_dir: Path
-
-    # Absolute file name of the archive.
-    #
-    # For example, 'X:\\TEMP\\FOO.ZIP'.
-    archive_filepath: Path
-
-    # Absolute name of a file which acts as an indication of the fact that the
-    # archive is ready and is available for access.
-    #
-    # This is how synchronization between buildbot worker and signing server is
-    # done:
-    # - First, the archive is created under archive_filepath name.
-    # - Second, the indication file is created under ready_indicator_filepath
-    #   name.
-    # - Third, the colleague of whoever created the indicator name watches for
-    #   the indication file to appear, and once it's there it access the
-    #   archive.
-    ready_indicator_filepath: Path
-
-    def __init__(
-            self, base_dir: Path, archive_name: str, ready_indicator_name: str):
-        """
-        Construct the object from given base directory and name of the archive
-        file:
-          ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY')
-        """
-
-        self.base_dir = base_dir
-        self.archive_filepath = self.base_dir / archive_name
-        self.ready_indicator_filepath = self.base_dir / ready_indicator_name
-
-    def is_ready_unsafe(self) -> bool:
-        """
-        Check whether the archive is ready for access.
-
-        No guarding about possible network failres is done here.
-        """
-        if not self.ready_indicator_filepath.exists():
-            return False
-
-        try:
-            archive_state = ArchiveState.deserialize_from_file(
-                self.ready_indicator_filepath)
-        except ArchiveStateError as error:
-            print(f'Error deserializing archive state: {error.message}')
-            return False
-
-        if archive_state.has_error():
-            # If the error did happen during codesign procedure there will be no
-            # corresponding archive file.
-            # The caller code will deal with the error check further.
-            return True
-
-        # Sometimes on macOS indicator file appears prior to the actual archive
-        # despite the order of creation and os.sync() used in tag_ready().
-        # So consider archive not ready if there is an indicator without an
-        # actual archive.
-        if not self.archive_filepath.exists():
-            print('Found indicator without actual archive, waiting for archive '
-                  f'({self.archive_filepath}) to appear.')
-            return False
-
-        # Wait for until archive is fully stored.
-        actual_archive_size = self.archive_filepath.stat().st_size
-        if actual_archive_size != archive_state.file_size:
-            print('Partial/invalid archive size (expected '
-                  f'{archive_state.file_size} got {actual_archive_size})')
-            return False
-
-        return True
-
-    def is_ready(self) -> bool:
-        """
-        Check whether the archive is ready for access.
-
-        Will tolerate possible network failures: if there is a network failure
-        or if there is still no proper permission on a file False is returned.
-        """
-
-        # There are some intermitten problem happening at a random which is
-        # translates to "OSError : [WinError 59] An unexpected network error occurred".
-        # Some reports suggests it might be due to lack of permissions to the file,
-        # which might be applicable in our case since it's possible that file is
-        # initially created with non-accessible permissions and gets chmod-ed
-        # after initial creation.
-        try:
-            return self.is_ready_unsafe()
-        except OSError as e:
-            print(f'Exception checking archive: {e}')
-            return False
-
-    def tag_ready(self, error_message='') -> None:
-        """
-        Tag the archive as ready by creating the corresponding indication file.
-
-        NOTE: It is expected that the archive was never tagged as ready before
-              and that there are no subsequent tags of the same archive.
-              If it is violated, an assert will fail.
-        """
-        assert not self.is_ready()
-
-        # Try the best to make sure everything is synced to the file system,
-        # to avoid any possibility of stamp appearing on a network share prior to
-        # an actual file.
-        if util.get_current_platform() != util.Platform.WINDOWS:
-            os.sync()
-
-        archive_size = -1
-        if self.archive_filepath.exists():
-            archive_size = self.archive_filepath.stat().st_size
-
-        archive_info = ArchiveState(
-            file_size=archive_size, error_message=error_message)
-
-        self.ready_indicator_filepath.write_text(
-            archive_info.serialize_to_string())
-
-    def get_state(self) -> ArchiveState:
-        """
-        Get state object for this archive
-
-        The state is read from the corresponding state file.
-        """
-
-        try:
-            return ArchiveState.deserialize_from_file(self.ready_indicator_filepath)
-        except ArchiveStateError as error:
-            return ArchiveState(error_message=f'Error in information format: {error}')
-
-    def clean(self) -> None:
-        """
-        Remove both archive and the ready indication file.
-        """
-        util.ensure_file_does_not_exist_or_die(self.ready_indicator_filepath)
-        util.ensure_file_does_not_exist_or_die(self.archive_filepath)
-
-    def is_fully_absent(self) -> bool:
-        """
-        Check whether both archive and its ready indicator are absent.
-        Is used for a sanity check during code signing process by both
-        buildbot worker and signing server.
-        """
-        return (not self.archive_filepath.exists() and
-                not self.ready_indicator_filepath.exists())

build_files/buildbot/codesign/base_code_signer.py

@@ -1,501 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Signing process overview.
-#
-# From buildbot worker side:
-#  - Files which needs to be signed are collected from either a directory to
-#    sign all signable files in there, or by filename of a single file to sign.
-#  - Those files gets packed into an archive and stored in a location location
-#    which is watched by the signing server.
-#  - A marker READY file is created which indicates the archive is ready for
-#    access.
-#  - Wait for the server to provide an archive with signed files.
-#    This is done by watching for the READY file which corresponds to an archive
-#    coming from the signing server.
-#  - Unpack the signed signed files from the archives and replace original ones.
-#
-# From code sign server:
-#  - Watch special location for a READY file which indicates the there is an
-#    archive with files which are to be signed.
-#  - Unpack the archive to a temporary location.
-#  - Run codesign tool and make sure all the files are signed.
-#  - Pack the signed files and store them in a location which is watched by
-#    the buildbot worker.
-#  - Create a READY file which indicates that the archive with signed files is
-#    ready.
-
-import abc
-import logging
-import shutil
-import subprocess
-import time
-import tarfile
-import uuid
-
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import Iterable, List
-
-import codesign.util as util
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.archive_with_indicator import ArchiveWithIndicator
-from codesign.exception import CodeSignException
-
-
-logger = logging.getLogger(__name__)
-logger_builder = logger.getChild('builder')
-logger_server = logger.getChild('server')
-
-
-def pack_files(files: Iterable[AbsoluteAndRelativeFileName],
-               archive_filepath: Path) -> None:
-    """
-    Create tar archive from given files for the signing pipeline.
-    Is used by buildbot worker to create an archive of files which are to be
-    signed, and by signing server to send signed files back to the worker.
-    """
-    with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle:
-        for file_info in files:
-            tar_file_handle.add(file_info.absolute_filepath,
-                                arcname=file_info.relative_filepath)
-
-
-def extract_files(archive_filepath: Path,
-                  extraction_dir: Path) -> None:
-    """
-    Extract all files form the given archive into the given direcotry.
-    """
-
-    # TODO(sergey): Verify files in the archive have relative path.
-
-    with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle:
-        tar_file_handle.extractall(path=extraction_dir)
-
-
-class BaseCodeSigner(metaclass=abc.ABCMeta):
-    """
-    Base class for a platform-specific signer of binaries.
-
-    Contains all the logic shared across platform-specific implementations, such
-    as synchronization and notification logic.
-
-    Platform specific bits (such as actual command for signing the binary) are
-    to be implemented as a subclass.
-
-    Provides utilities code signing as a whole, including functionality needed
-    by a signing server and a buildbot worker.
-
-    The signer and builder may run on separate machines, the only requirement is
-    that they have access to a directory which is shared between them. For the
-    security concerns this is to be done as a separate machine (or as a Shared
-    Folder configuration in VirtualBox configuration). This directory might be
-    mounted under different base paths, but its underlying storage is to be
-    the same.
-
-    The code signer is short-lived on a buildbot worker side, and is living
-    forever on a code signing server side.
-    """
-
-    # TODO(sergey): Find a neat way to have config annotated.
-    # config: Config
-
-    # Storage directory where builder puts files which are requested to be
-    # signed.
-    # Consider this an input of the code signing server.
-    unsigned_storage_dir: Path
-
-    # Storage where signed files are stored.
-    # Consider this an output of the code signer server.
-    signed_storage_dir: Path
-
-    # Platform the code is currently executing on.
-    platform: util.Platform
-
-    def __init__(self, config):
-        self.config = config
-
-        absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve()
-
-        # Unsigned (signing server input) configuration.
-        self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned'
-
-        # Signed (signing server output) configuration.
-        self.signed_storage_dir = absolute_shared_storage_dir / 'signed'
-
-        self.platform = util.get_current_platform()
-
-    def cleanup_environment_for_builder(self) -> None:
-        # TODO(sergey): Revisit need of cleaning up the existing files.
-        # In practice it wasn't so helpful, and with multiple clients
-        # talking to the same server it becomes even more tricky.
-        pass
-
-    def cleanup_environment_for_signing_server(self) -> None:
-        # TODO(sergey): Revisit need of cleaning up the existing files.
-        # In practice it wasn't so helpful, and with multiple clients
-        # talking to the same server it becomes even more tricky.
-        pass
-
-    def generate_request_id(self) -> str:
-        """
-        Generate an unique identifier for code signing request.
-        """
-        return str(uuid.uuid4())
-
-    def archive_info_for_request_id(
-            self, path: Path, request_id: str) -> ArchiveWithIndicator:
-        return ArchiveWithIndicator(
-            path, f'{request_id}.tar', f'{request_id}.ready')
-
-    def signed_archive_info_for_request_id(
-            self, request_id: str) -> ArchiveWithIndicator:
-        return self.archive_info_for_request_id(
-            self.signed_storage_dir, request_id)
-
-    def unsigned_archive_info_for_request_id(
-            self, request_id: str) -> ArchiveWithIndicator:
-        return self.archive_info_for_request_id(
-            self.unsigned_storage_dir, request_id)
-
-    ############################################################################
-    # Buildbot worker side helpers.
-
-    @abc.abstractmethod
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        """
-        Check whether file is to be signed.
-
-        Is used by both single file signing pipeline and recursive directory
-        signing pipeline.
-
-        This is where code signer is to check whether file is to be signed or
-        not. This check might be based on a simple extension test or on actual
-        test whether file have a digital signature already or not.
-        """
-
-    def collect_files_to_sign(self, path: Path) \
-            -> List[AbsoluteAndRelativeFileName]:
-        """
-        Get all files which need to be signed from the given path.
-
-        NOTE: The path might either be a file or directory.
-
-        This function is run from the buildbot worker side.
-        """
-
-        # If there is a single file provided trust the buildbot worker that it
-        # is eligible for signing.
-        if path.is_file():
-            file = AbsoluteAndRelativeFileName.from_path(path)
-            if not self.check_file_is_to_be_signed(file):
-                return []
-            return [file]
-
-        all_files = AbsoluteAndRelativeFileName.recursively_from_directory(
-            path)
-        files_to_be_signed = [file for file in all_files
-                              if self.check_file_is_to_be_signed(file)]
-        return files_to_be_signed
-
-    def wait_for_signed_archive_or_die(self, request_id) -> None:
-        """
-        Wait until archive with signed files is available.
-
-        Will only return if the archive with signed files is available. If there
-        was an error during code sign procedure the SystemExit exception is
-        raised, with the message set to the error reported by the codesign
-        server.
-
-        Will only wait for the configured time. If that time exceeds and there
-        is still no responce from the signing server the application will exit
-        with a non-zero exit code.
-
-        """
-
-        signed_archive_info = self.signed_archive_info_for_request_id(
-            request_id)
-        unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-            request_id)
-
-        timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS
-        time_start = time.monotonic()
-        while not signed_archive_info.is_ready():
-            time.sleep(1)
-            time_slept_in_seconds = time.monotonic() - time_start
-            if time_slept_in_seconds > timeout_in_seconds:
-                signed_archive_info.clean()
-                unsigned_archive_info.clean()
-                raise SystemExit("Signing server didn't finish signing in "
-                                 f'{timeout_in_seconds} seconds, dying :(')
-
-        archive_state = signed_archive_info.get_state()
-        if archive_state.has_error():
-            signed_archive_info.clean()
-            unsigned_archive_info.clean()
-            raise SystemExit(
-                f'Error happenned during codesign procedure: {archive_state.error_message}')
-
-    def copy_signed_files_to_directory(
-            self, signed_dir: Path, destination_dir: Path) -> None:
-        """
-        Copy all files from signed_dir to destination_dir.
-
-        This function will overwrite any existing file. Permissions are copied
-        from the source files, but other metadata, such as timestamps, are not.
-        """
-        for signed_filepath in signed_dir.glob('**/*'):
-            if not signed_filepath.is_file():
-                continue
-
-            relative_filepath = signed_filepath.relative_to(signed_dir)
-            destination_filepath = destination_dir / relative_filepath
-            destination_filepath.parent.mkdir(parents=True, exist_ok=True)
-
-            shutil.copy(signed_filepath, destination_filepath)
-
-    def run_buildbot_path_sign_pipeline(self, path: Path) -> None:
-        """
-        Run all steps needed to make given path signed.
-
-        Path points to an unsigned file or a directory which contains unsigned
-        files.
-
-        If the path points to a single file then this file will be signed.
-        This is used to sign a final bundle such as .msi on Windows or .dmg on
-        macOS.
-
-        NOTE: The code signed implementation might actually reject signing the
-        file, in which case the file will be left unsigned. This isn't anything
-        to be considered a failure situation, just might happen when buildbot
-        worker can not detect whether signing is really required in a specific
-        case or not.
-
-        If the path points to a directory then code signer will sign all
-        signable files from it (finding them recursively).
-        """
-
-        self.cleanup_environment_for_builder()
-
-        # Make sure storage directory exists.
-        self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True)
-
-        # Collect all files which needs to be signed and pack them into a single
-        # archive which will be sent to the signing server.
-        logger_builder.info('Collecting files which are to be signed...')
-        files = self.collect_files_to_sign(path)
-        if not files:
-            logger_builder.info('No files to be signed, ignoring.')
-            return
-        logger_builder.info('Found %d files to sign.', len(files))
-
-        request_id = self.generate_request_id()
-        signed_archive_info = self.signed_archive_info_for_request_id(
-            request_id)
-        unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-            request_id)
-
-        pack_files(files=files,
-                   archive_filepath=unsigned_archive_info.archive_filepath)
-        unsigned_archive_info.tag_ready()
-
-        # Wait for the signing server to finish signing.
-        logger_builder.info('Waiting signing server to sign the files...')
-        self.wait_for_signed_archive_or_die(request_id)
-
-        # Extract signed files from archive and move files to final location.
-        with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str:
-            unpacked_signed_files_dir = Path(temp_dir_str)
-
-            logger_builder.info('Extracting signed files from archive...')
-            extract_files(
-                archive_filepath=signed_archive_info.archive_filepath,
-                extraction_dir=unpacked_signed_files_dir)
-
-            destination_dir = path
-            if destination_dir.is_file():
-                destination_dir = destination_dir.parent
-            self.copy_signed_files_to_directory(
-                unpacked_signed_files_dir, destination_dir)
-
-        logger_builder.info('Removing archive with signed files...')
-        signed_archive_info.clean()
-
-    ############################################################################
-    # Signing server side helpers.
-
-    def wait_for_sign_request(self) -> str:
-        """
-        Wait for the buildbot to request signing of an archive.
-
-        Returns an identifier of signing request.
-        """
-
-        # TOOD(sergey): Support graceful shutdown on Ctrl-C.
-
-        logger_server.info(
-            f'Waiting for a request directory {self.unsigned_storage_dir} to appear.')
-        while not self.unsigned_storage_dir.exists():
-            time.sleep(1)
-
-        logger_server.info(
-            'Waiting for a READY indicator of any signing request.')
-        request_id = None
-        while request_id is None:
-            for file in self.unsigned_storage_dir.iterdir():
-                if file.suffix != '.ready':
-                    continue
-                request_id = file.stem
-                logger_server.info(f'Found READY for request ID {request_id}.')
-            if request_id is None:
-                time.sleep(1)
-
-        unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-            request_id)
-        while not unsigned_archive_info.is_ready():
-            time.sleep(1)
-
-        return request_id
-
-    @abc.abstractmethod
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        """
-        Sign all files in the given directory.
-
-        NOTE: Signing should happen in-place.
-        """
-
-    def run_signing_pipeline(self, request_id: str):
-        """
-        Run the full signing pipeline starting from the point when buildbot
-        worker have requested signing.
-        """
-
-        # Make sure storage directory exists.
-        self.signed_storage_dir.mkdir(parents=True, exist_ok=True)
-
-        with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str:
-            temp_dir = Path(temp_dir_str)
-
-            signed_archive_info = self.signed_archive_info_for_request_id(
-                request_id)
-            unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-                request_id)
-
-            logger_server.info('Extracting unsigned files from archive...')
-            extract_files(
-                archive_filepath=unsigned_archive_info.archive_filepath,
-                extraction_dir=temp_dir)
-
-            logger_server.info('Collecting all files which needs signing...')
-            files = AbsoluteAndRelativeFileName.recursively_from_directory(
-                temp_dir)
-
-            logger_server.info('Signing all requested files...')
-            try:
-                self.sign_all_files(files)
-            except CodeSignException as error:
-                signed_archive_info.tag_ready(error_message=error.message)
-                unsigned_archive_info.clean()
-                logger_server.info('Signing is complete with errors.')
-                return
-
-            logger_server.info('Packing signed files...')
-            pack_files(files=files,
-                       archive_filepath=signed_archive_info.archive_filepath)
-            signed_archive_info.tag_ready()
-
-            logger_server.info('Removing signing request...')
-            unsigned_archive_info.clean()
-
-            logger_server.info('Signing is complete.')
-
-    def run_signing_server(self):
-        logger_server.info('Starting new code signing server...')
-        self.cleanup_environment_for_signing_server()
-        logger_server.info('Code signing server is ready')
-        while True:
-            logger_server.info('Waiting for the signing request in %s...',
-                               self.unsigned_storage_dir)
-            request_id = self.wait_for_sign_request()
-
-            logger_server.info(
-                f'Beging signign procedure for request ID {request_id}.')
-            self.run_signing_pipeline(request_id)
-
-    ############################################################################
-    # Command executing.
-    #
-    # Abstracted to a degree that allows to run commands from a foreign
-    # platform.
-    # The goal with this is to allow performing dry-run tests of code signer
-    # server from other platforms (for example, to test that macOS code signer
-    # does what it is supposed to after doing a refactor on Linux).
-
-    # TODO(sergey): What is the type annotation for the command?
-    def run_command_or_mock(self, command, platform: util.Platform) -> None:
-        """
-        Run given command if current platform matches given one
-
-        If the platform is different then it will only be printed allowing
-        to verify logic of the code signing process.
-        """
-
-        if platform != self.platform:
-            logger_server.info(
-                f'Will run command for {platform}: {command}')
-            return
-
-        logger_server.info(f'Running command: {command}')
-        subprocess.run(command)
-
-    # TODO(sergey): What is the type annotation for the command?
-    def check_output_or_mock(self, command,
-                             platform: util.Platform,
-                             allow_nonzero_exit_code=False) -> str:
-        """
-        Run given command if current platform matches given one
-
-        If the platform is different then it will only be printed allowing
-        to verify logic of the code signing process.
-
-        If allow_nonzero_exit_code is truth then the output will be returned
-        even if application quit with non-zero exit code.
-        Otherwise an subprocess.CalledProcessError exception will be raised
-        in such case.
-        """
-
-        if platform != self.platform:
-            logger_server.info(
-                f'Will run command for {platform}: {command}')
-            return
-
-        if allow_nonzero_exit_code:
-            process = subprocess.Popen(command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.STDOUT)
-            output = process.communicate()[0]
-            return output.decode()
-
-        logger_server.info(f'Running command: {command}')
-        return subprocess.check_output(
-            command, stderr=subprocess.STDOUT).decode()

build_files/buildbot/codesign/config_builder.py

@@ -1,62 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Configuration of a code signer which is specific to the code running from
-# buildbot's worker.
-
-import sys
-
-from pathlib import Path
-
-import codesign.util as util
-
-from codesign.config_common import *
-
-platform = util.get_current_platform()
-if platform == util.Platform.LINUX:
-    SHARED_STORAGE_DIR = Path('/data/codesign')
-elif platform == util.Platform.WINDOWS:
-    SHARED_STORAGE_DIR = Path('Z:\\codesign')
-elif platform == util.Platform.MACOS:
-    SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign')
-
-# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
-LOGGING = {
-    'version': 1,
-    'formatters': {
-        'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
-    },
-    'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
-            'formatter': 'default',
-            'stream': 'ext://sys.stderr',
-        }
-    },
-    'loggers': {
-        'codesign': {'level': 'INFO'},
-    },
-    'root': {
-        'level': 'WARNING',
-        'handlers': [
-            'console',
-        ],
-    }
-}

build_files/buildbot/codesign/config_common.py

@@ -1,36 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-from pathlib import Path
-
-# Timeout in seconds for the signing process.
-#
-# This is how long buildbot packing step will wait signing server to
-# perform signing.
-#
-# NOTE: Notarization could take a long time, hence the rather high value
-# here. Might consider using different timeout for different platforms.
-TIMEOUT_IN_SECONDS = 45 * 60 * 60
-
-# Directory which is shared across buildbot worker and signing server.
-#
-# This is where worker puts files requested for signing as well as where
-# server puts signed files.
-SHARED_STORAGE_DIR: Path

build_files/buildbot/codesign/config_server_template.py

@@ -1,101 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Configuration of a code signer which is specific to the code signing server.
-#
-# NOTE: DO NOT put any sensitive information here, put it in an actual
-# configuration on the signing machine.
-
-from pathlib import Path
-
-from codesign.config_common import *
-
-CODESIGN_DIRECTORY = Path(__file__).absolute().parent
-BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent
-
-################################################################################
-# Common configuration.
-
-# Directory where folders for codesign requests and signed result are stored.
-# For example, /data/codesign
-SHARED_STORAGE_DIR: Path
-
-################################################################################
-# macOS-specific configuration.
-
-MACOS_ENTITLEMENTS_FILE = \
-    BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist'
-
-# Identity of the Developer ID Application certificate which is to be used for
-# codesign tool.
-# Use `security find-identity -v -p codesigning` to find the identity.
-#
-# NOTE: This identity is just an example from release/darwin/README.txt.
-MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97'
-
-# User name (Apple ID) which will be used to request notarization.
-MACOS_XCRUN_USERNAME = 'me@example.com'
-
-# One-time application password which will be used to request notarization.
-MACOS_XCRUN_PASSWORD = '@keychain:altool-password'
-
-# Timeout in seconds within which the notarial office is supposed to reply.
-MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60
-
-################################################################################
-# Windows-specific configuration.
-
-# URL to the timestamping authority.
-WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com'
-
-# Full path to the certificate used for signing.
-#
-# The path and expected file format might vary depending on a platform.
-#
-# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look
-# like Path('C:\\Secret\\Blender.pfx').
-WIN_CERTIFICATE_FILEPATH: Path
-
-################################################################################
-# Logging configuration, common for all platforms.
-
-# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
-LOGGING = {
-    'version': 1,
-    'formatters': {
-        'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
-    },
-    'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
-            'formatter': 'default',
-            'stream': 'ext://sys.stderr',
-        }
-    },
-    'loggers': {
-        'codesign': {'level': 'INFO'},
-    },
-    'root': {
-        'level': 'WARNING',
-        'handlers': [
-            'console',
-        ],
-    }
-}

build_files/buildbot/codesign/exception.py

@@ -1,26 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-class CodeSignException(Exception):
-    message: str
-
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)

build_files/buildbot/codesign/linux_code_signer.py

@@ -1,72 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# NOTE: This is a no-op signer (since there isn't really a procedure to sign
-# Linux binaries yet). Used to debug and verify the code signing routines on
-# a Linux environment.
-
-import logging
-
-from pathlib import Path
-from typing import List
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.base_code_signer import BaseCodeSigner
-
-logger = logging.getLogger(__name__)
-logger_server = logger.getChild('server')
-
-
-class LinuxCodeSigner(BaseCodeSigner):
-    def is_active(self) -> bool:
-        """
-        Check whether this signer is active.
-
-        if it is inactive, no files will be signed.
-
-        Is used to be able to debug code signing pipeline on Linux, where there
-        is no code signing happening in the actual buildbot and release
-        environment.
-        """
-        return False
-
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        if file.relative_filepath == Path('blender'):
-            return True
-        if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and
-                file.relative_filepath.name.startwith('python')):
-            return True
-        if file.relative_filepath.suffix == '.so':
-            return True
-        return False
-
-    def collect_files_to_sign(self, path: Path) \
-            -> List[AbsoluteAndRelativeFileName]:
-        if not self.is_active():
-            return []
-
-        return super().collect_files_to_sign(path)
-
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        num_files = len(files)
-        for file_index, file in enumerate(files):
-            logger.info('Server: Signed file [%d/%d] %s',
-                        file_index + 1, num_files, file.relative_filepath)

build_files/buildbot/codesign/macos_code_signer.py

@@ -1,456 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import logging
-import re
-import stat
-import subprocess
-import time
-
-from pathlib import Path
-from typing import List
-
-import codesign.util as util
-
-from buildbot_utils import Builder
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.base_code_signer import BaseCodeSigner
-from codesign.exception import CodeSignException
-
-logger = logging.getLogger(__name__)
-logger_server = logger.getChild('server')
-
-# NOTE: Check is done as filename.endswith(), so keep the dot
-EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'}
-
-# Prefixes of a file (not directory) name which are to be signed.
-# Used to sign extra executable files in Contents/Resources.
-NAME_PREFIXES_TO_BE_SIGNED = {'python'}
-
-
-class NotarizationException(CodeSignException):
-    pass
-
-
-def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool:
-    """
-    Check whether file is coming from an .app bundle
-    """
-    parts = file.relative_filepath.parts
-    if not parts:
-        return False
-    if not parts[0].endswith('.app'):
-        return False
-    return True
-
-
-def get_bundle_from_file(
-        file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName:
-    """
-    Get AbsoluteAndRelativeFileName descriptor of bundle
-    """
-    assert(is_file_from_bundle(file))
-
-    parts = file.relative_filepath.parts
-    bundle_name = parts[0]
-
-    base_dir = file.base_dir
-    bundle_filepath = file.base_dir / bundle_name
-    return AbsoluteAndRelativeFileName(base_dir, bundle_filepath)
-
-
-def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool:
-    """
-    Check whether given file is an executable within an app bundle
-    """
-    if not is_file_from_bundle(file):
-        return False
-
-    parts = file.relative_filepath.parts
-    num_parts = len(parts)
-    if num_parts < 3:
-        return False
-
-    if parts[1:3] != ('Contents', 'MacOS'):
-        return False
-
-    return True
-
-
-def xcrun_field_value_from_output(field: str, output: str) -> str:
-    """
-    Get value of a given field from xcrun output.
-
-    If field is not found empty string is returned.
-    """
-
-    field_prefix = field + ': '
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith(field_prefix):
-            return line[len(field_prefix):]
-    return ''
-
-
-class MacOSCodeSigner(BaseCodeSigner):
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        if file.relative_filepath.name.startswith('.'):
-            return False
-
-        if is_bundle_executable_file(file):
-            return True
-
-        base_name = file.relative_filepath.name
-        if any(base_name.startswith(prefix)
-               for prefix in NAME_PREFIXES_TO_BE_SIGNED):
-            return True
-
-        mode = file.absolute_filepath.lstat().st_mode
-        if mode & stat.S_IXUSR != 0:
-            file_output = subprocess.check_output(
-                ("file", file.absolute_filepath)).decode()
-            if "64-bit executable" in file_output:
-                return True
-
-        return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED
-
-    def collect_files_to_sign(self, path: Path) \
-            -> List[AbsoluteAndRelativeFileName]:
-        # Include all files when signing app or dmg bundle: all the files are
-        # needed to do valid signature of bundle.
-        if path.name.endswith('.app'):
-            return AbsoluteAndRelativeFileName.recursively_from_directory(path)
-        if path.is_dir():
-            files = []
-            for child in path.iterdir():
-                if child.name.endswith('.app'):
-                    current_files = AbsoluteAndRelativeFileName.recursively_from_directory(
-                        child)
-                else:
-                    current_files = super().collect_files_to_sign(child)
-                for current_file in current_files:
-                    files.append(AbsoluteAndRelativeFileName(
-                        path, current_file.absolute_filepath))
-            return files
-        return super().collect_files_to_sign(path)
-
-    ############################################################################
-    # Codesign.
-
-    def codesign_remove_signature(
-            self, file: AbsoluteAndRelativeFileName) -> None:
-        """
-        Make sure given file does not have codesign signature
-
-        This is needed because codesigning is not possible for file which has
-        signature already.
-        """
-
-        logger_server.info(
-            'Removing codesign signature from %s...', file.relative_filepath)
-
-        command = ['codesign', '--remove-signature', file.absolute_filepath]
-        self.run_command_or_mock(command, util.Platform.MACOS)
-
-    def codesign_file(
-            self, file: AbsoluteAndRelativeFileName) -> None:
-        """
-        Sign given file
-
-        NOTE: File must not have any signatures.
-        """
-
-        logger_server.info(
-            'Codesigning %s...', file.relative_filepath)
-
-        entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE
-        command = ['codesign',
-                   '--timestamp',
-                   '--options', 'runtime',
-                   f'--entitlements={entitlements_file}',
-                   '--sign', self.config.MACOS_CODESIGN_IDENTITY,
-                   file.absolute_filepath]
-        self.run_command_or_mock(command, util.Platform.MACOS)
-
-    def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        """
-        Run codesign tool on all eligible files in the given list.
-
-        Will ignore all files which are not to be signed. For the rest will
-        remove possible existing signature and add a new signature.
-        """
-
-        num_files = len(files)
-        have_ignored_files = False
-        signed_files = []
-        for file_index, file in enumerate(files):
-            # Ignore file if it is not to be signed.
-            # Allows to manually construct ZIP of a bundle and get it signed.
-            if not self.check_file_is_to_be_signed(file):
-                logger_server.info(
-                    'Ignoring file [%d/%d] %s',
-                    file_index + 1, num_files, file.relative_filepath)
-                have_ignored_files = True
-                continue
-
-            logger_server.info(
-                'Running codesigning routines for file [%d/%d] %s...',
-                file_index + 1, num_files, file.relative_filepath)
-
-            self.codesign_remove_signature(file)
-            self.codesign_file(file)
-
-            signed_files.append(file)
-
-        if have_ignored_files:
-            logger_server.info('Signed %d files:', len(signed_files))
-            num_signed_files = len(signed_files)
-            for file_index, signed_file in enumerate(signed_files):
-                logger_server.info(
-                    '- [%d/%d] %s',
-                    file_index + 1, num_signed_files,
-                    signed_file.relative_filepath)
-
-    def codesign_bundles(
-            self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        """
-        Codesign all .app bundles in the given list of files.
-
-        Bundle is deducted from paths of the files, and every bundle is only
-        signed once.
-        """
-
-        signed_bundles = set()
-        extra_files = []
-
-        for file in files:
-            if not is_file_from_bundle(file):
-                continue
-            bundle = get_bundle_from_file(file)
-            bundle_name = bundle.relative_filepath
-            if bundle_name in signed_bundles:
-                continue
-
-            logger_server.info('Running codesign routines on bundle %s',
-                               bundle_name)
-
-            # It is not possible to remove signature from DMG.
-            if bundle.relative_filepath.name.endswith('.app'):
-                self.codesign_remove_signature(bundle)
-            self.codesign_file(bundle)
-
-            signed_bundles.add(bundle_name)
-
-            # Codesign on a bundle adds an extra folder with information.
-            # It needs to be compied to the source.
-            code_signature_directory = \
-                bundle.absolute_filepath / 'Contents' / '_CodeSignature'
-            code_signature_files = \
-                AbsoluteAndRelativeFileName.recursively_from_directory(
-                    code_signature_directory)
-            for code_signature_file in code_signature_files:
-                bundle_relative_file = AbsoluteAndRelativeFileName(
-                    bundle.base_dir,
-                    code_signature_directory /
-                    code_signature_file.relative_filepath)
-                extra_files.append(bundle_relative_file)
-
-        files.extend(extra_files)
-
-    ############################################################################
-    # Notarization.
-
-    def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str:
-        """
-        Get bundle ID which will be used to notarize DMG
-        """
-        name = file.relative_filepath.name
-        app_name = name.split('-', 2)[0].lower()
-
-        app_name_words = app_name.split()
-        if len(app_name_words) > 1:
-            app_name_id = ''.join(word.capitalize() for word in app_name_words)
-        else:
-            app_name_id = app_name_words[0]
-
-        # TODO(sergey): Consider using "alpha" for buildbot builds.
-        return f'org.blenderfoundation.{app_name_id}.release'
-
-    def notarize_request(self, file) -> str:
-        """
-        Request notarization of the given file.
-
-        Returns UUID of the notarization request. If error occurred None is
-        returned instead of UUID.
-        """
-
-        bundle_id = self.notarize_get_bundle_id(file)
-        logger_server.info('Bundle ID: %s', bundle_id)
-
-        logger_server.info('Submitting file to the notarial office.')
-        command = [
-            'xcrun', 'altool', '--notarize-app', '--verbose',
-            '-f', file.absolute_filepath,
-            '--primary-bundle-id', bundle_id,
-            '--username', self.config.MACOS_XCRUN_USERNAME,
-            '--password', self.config.MACOS_XCRUN_PASSWORD]
-
-        output = self.check_output_or_mock(
-            command, util.Platform.MACOS, allow_nonzero_exit_code=True)
-
-        for line in output.splitlines():
-            line = line.strip()
-            if line.startswith('RequestUUID = '):
-                request_uuid = line[14:]
-                return request_uuid
-
-            # Check whether the package has been already submitted.
-            if 'The software asset has already been uploaded.' in line:
-                request_uuid = re.sub(
-                    '.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line)
-                logger_server.warning(
-                    f'The package has been already submitted under UUID {request_uuid}')
-                return request_uuid
-
-        logger_server.error(output)
-        logger_server.error('xcrun command did not report RequestUUID')
-        return None
-
-    def notarize_review_status(self, xcrun_output: str) -> bool:
-        """
-        Review status returned by xcrun's notarization info
-
-        Returns truth if the notarization process has finished.
-        If there are errors during notarization, a NotarizationException()
-        exception is thrown with status message from the notarial office.
-        """
-
-        # Parse status and message
-        status = xcrun_field_value_from_output('Status', xcrun_output)
-        status_message = xcrun_field_value_from_output(
-            'Status Message', xcrun_output)
-
-        if status == 'success':
-            logger_server.info(
-                'Package successfully notarized: %s', status_message)
-            return True
-
-        if status == 'invalid':
-            logger_server.error(xcrun_output)
-            logger_server.error(
-                'Package notarization has failed: %s', status_message)
-            raise NotarizationException(status_message)
-
-        if status == 'in progress':
-            return False
-
-        logger_server.info(
-            'Unknown notarization status %s (%s)', status, status_message)
-
-        return False
-
-    def notarize_wait_result(self, request_uuid: str) -> None:
-        """
-        Wait for until notarial office have a reply
-        """
-
-        logger_server.info(
-            'Waiting for a result from the notarization office.')
-
-        command = ['xcrun', 'altool',
-                   '--notarization-info', request_uuid,
-                   '--username', self.config.MACOS_XCRUN_USERNAME,
-                   '--password', self.config.MACOS_XCRUN_PASSWORD]
-
-        time_start = time.monotonic()
-        timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS
-
-        while True:
-            xcrun_output = self.check_output_or_mock(
-                command, util.Platform.MACOS, allow_nonzero_exit_code=True)
-
-            if self.notarize_review_status(xcrun_output):
-                break
-
-            logger_server.info('Keep waiting for notarization office.')
-            time.sleep(30)
-
-            time_slept_in_seconds = time.monotonic() - time_start
-            if time_slept_in_seconds > timeout_in_seconds:
-                logger_server.error(
-                    "Notarial office didn't reply in %f seconds.",
-                    timeout_in_seconds)
-
-    def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool:
-        """
-        Staple notarial label on the file
-        """
-
-        logger_server.info('Stapling notarial stamp.')
-
-        command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath]
-        self.check_output_or_mock(command, util.Platform.MACOS)
-
-    def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool:
-        """
-        Run entire pipeline to get DMG notarized.
-        """
-        logger_server.info('Begin notarization routines on %s',
-                           file.relative_filepath)
-
-        # Submit file for notarization.
-        request_uuid = self.notarize_request(file)
-        if not request_uuid:
-            return False
-        logger_server.info('Received Request UUID: %s', request_uuid)
-
-        # Wait for the status from the notarization office.
-        if not self.notarize_wait_result(request_uuid):
-            return False
-
-        # Staple.
-        self.notarize_staple(file)
-
-    def notarize_all_dmg(
-            self, files: List[AbsoluteAndRelativeFileName]) -> bool:
-        """
-        Notarize all DMG images from the input.
-
-        Images are supposed to be codesigned already.
-        """
-        for file in files:
-            if not file.relative_filepath.name.endswith('.dmg'):
-                continue
-            if not self.check_file_is_to_be_signed(file):
-                continue
-
-            self.notarize_dmg(file)
-
-    ############################################################################
-    # Entry point.
-
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        # TODO(sergey): Handle errors somehow.
-
-        self.codesign_all_files(files)
-        self.codesign_bundles(files)
-        self.notarize_all_dmg(files)

build_files/buildbot/codesign/simple_code_signer.py

@@ -1,52 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-
-import logging.config
-import sys
-
-from pathlib import Path
-from typing import Optional
-
-import codesign.config_builder
-import codesign.util as util
-from codesign.base_code_signer import BaseCodeSigner
-
-
-class SimpleCodeSigner:
-    code_signer: Optional[BaseCodeSigner]
-
-    def __init__(self):
-        platform = util.get_current_platform()
-        if platform == util.Platform.LINUX:
-            from codesign.linux_code_signer import LinuxCodeSigner
-            self.code_signer = LinuxCodeSigner(codesign.config_builder)
-        elif platform == util.Platform.MACOS:
-            from codesign.macos_code_signer import MacOSCodeSigner
-            self.code_signer = MacOSCodeSigner(codesign.config_builder)
-        elif platform == util.Platform.WINDOWS:
-            from codesign.windows_code_signer import WindowsCodeSigner
-            self.code_signer = WindowsCodeSigner(codesign.config_builder)
-        else:
-            self.code_signer = None
-
-    def sign_file_or_directory(self, path: Path) -> None:
-        logging.config.dictConfig(codesign.config_builder.LOGGING)
-        self.code_signer.run_buildbot_path_sign_pipeline(path)

build_files/buildbot/codesign/util.py

@@ -1,54 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import sys
-
-from enum import Enum
-from pathlib import Path
-
-
-class Platform(Enum):
-    LINUX = 1
-    MACOS = 2
-    WINDOWS = 3
-
-
-def get_current_platform() -> Platform:
-    if sys.platform == 'linux':
-        return Platform.LINUX
-    elif sys.platform == 'darwin':
-        return Platform.MACOS
-    elif sys.platform == 'win32':
-        return Platform.WINDOWS
-    raise Exception(f'Unknown platform {sys.platform}')
-
-
-def ensure_file_does_not_exist_or_die(filepath: Path) -> None:
-    """
-    If the file exists, unlink it.
-    If the file path exists and is not a file an assert will trigger.
-    If the file path does not exists nothing happens.
-    """
-    if not filepath.exists():
-        return
-    if not filepath.is_file():
-        # TODO(sergey): Provide information about what the filepath actually is.
-        raise SystemExit(f'{filepath} is expected to be a file, but is not')
-    filepath.unlink()

build_files/buildbot/codesign/windows_code_signer.py

@@ -1,117 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import logging
-
-from pathlib import Path
-from typing import List
-
-import codesign.util as util
-
-from buildbot_utils import Builder
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.base_code_signer import BaseCodeSigner
-from codesign.exception import CodeSignException
-
-logger = logging.getLogger(__name__)
-logger_server = logger.getChild('server')
-
-# NOTE: Check is done as filename.endswith(), so keep the dot
-EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'}
-
-BLACKLIST_FILE_PREFIXES = (
-    'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime')
-
-
-class SigntoolException(CodeSignException):
-    pass
-
-class WindowsCodeSigner(BaseCodeSigner):
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        base_name = file.relative_filepath.name
-        if any(base_name.startswith(prefix)
-               for prefix in BLACKLIST_FILE_PREFIXES):
-            return False
-
-        return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED
-
-
-    def get_sign_command_prefix(self) -> List[str]:
-        return [
-            'signtool', 'sign', '/v',
-            '/f', self.config.WIN_CERTIFICATE_FILEPATH,
-            '/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL]
-
-
-    def run_codesign_tool(self, filepath: Path) -> None:
-        command = self.get_sign_command_prefix() + [filepath]
-
-        try:
-            codesign_output = self.check_output_or_mock(command, util.Platform.WINDOWS)
-        except subprocess.CalledProcessError as e:
-            raise SigntoolException(f'Error running signtool {e}')
-
-        logger_server.info(f'signtool output:\n{codesign_output}')
-
-        got_number_of_success = False
-
-        for line in codesign_output.split('\n'):
-            line_clean = line.strip()
-            line_clean_lower = line_clean.lower()
-
-            if line_clean_lower.startswith('number of warnings') or \
-               line_clean_lower.startswith('number of errors'):
-                 number = int(line_clean_lower.split(':')[1])
-                 if number != 0:
-                     raise SigntoolException('Non-clean success of signtool')
-
-            if line_clean_lower.startswith('number of files successfully signed'):
-                 got_number_of_success = True
-                 number = int(line_clean_lower.split(':')[1])
-                 if number != 1:
-                     raise SigntoolException('Signtool did not consider codesign a success')
-
-        if not got_number_of_success:
-            raise SigntoolException('Signtool did not report number of files signed')
-
-
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        # NOTE: Sign files one by one to avoid possible command line length
-        # overflow (which could happen if we ever decide to sign every binary
-        # in the install folder, for example).
-        #
-        # TODO(sergey): Consider doing batched signing of handful of files in
-        # one go (but only if this actually known to be much faster).
-        num_files = len(files)
-        for file_index, file in enumerate(files):
-            # Ignore file if it is not to be signed.
-            # Allows to manually construct ZIP of package and get it signed.
-            if not self.check_file_is_to_be_signed(file):
-                logger_server.info(
-                    'Ignoring file [%d/%d] %s',
-                    file_index + 1, num_files, file.relative_filepath)
-                continue
-
-            logger_server.info(
-                'Running signtool command for file [%d/%d] %s...',
-                file_index + 1, num_files, file.relative_filepath)
-            self.run_codesign_tool(file.absolute_filepath)

build_files/buildbot/codesign_server_linux.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# NOTE: This is a no-op signer (since there isn't really a procedure to sign
-# Linux binaries yet). Used to debug and verify the code signing routines on
-# a Linux environment.
-
-import logging.config
-from pathlib import Path
-from typing import List
-
-from codesign.linux_code_signer import LinuxCodeSigner
-import codesign.config_server
-
-if __name__ == "__main__":
-    logging.config.dictConfig(codesign.config_server.LOGGING)
-    code_signer = LinuxCodeSigner(codesign.config_server)
-    code_signer.run_signing_server()

build_files/buildbot/codesign_server_macos.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import logging.config
-from pathlib import Path
-from typing import List
-
-from codesign.macos_code_signer import MacOSCodeSigner
-import codesign.config_server
-
-if __name__ == "__main__":
-    entitlements_file = codesign.config_server.MACOS_ENTITLEMENTS_FILE
-    if not entitlements_file.exists():
-        raise SystemExit(
-            'Entitlements file {entitlements_file} does not exist.')
-    if not entitlements_file.is_file():
-        raise SystemExit(
-            'Entitlements file {entitlements_file} is not a file.')
-
-    logging.config.dictConfig(codesign.config_server.LOGGING)
-    code_signer = MacOSCodeSigner(codesign.config_server)
-    code_signer.run_signing_server()

build_files/buildbot/codesign_server_windows.bat

@@ -1,11 +0,0 @@
-@echo off
-
-rem This is an entry point of the codesign server for Windows.
-rem It makes sure that signtool.exe is within the current PATH and can be
-rem used by the Python script.
-
-SETLOCAL
-
-set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH%
-
-codesign_server_windows.py

build_files/buildbot/codesign_server_windows.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Implementation of codesign server for Windows.
-#
-# NOTE: If signtool.exe is not in the PATH use codesign_server_windows.bat
-
-import logging.config
-import shutil
-
-from pathlib import Path
-from typing import List
-
-import codesign.util as util
-
-from codesign.windows_code_signer import WindowsCodeSigner
-import codesign.config_server
-
-if __name__ == "__main__":
-    logging.config.dictConfig(codesign.config_server.LOGGING)
-
-    logger = logging.getLogger(__name__)
-    logger_server = logger.getChild('server')
-
-    # TODO(sergey): Consider moving such sanity checks into
-    # CodeSigner.check_environment_or_die().
-    if not shutil.which('signtool.exe'):
-        if util.get_current_platform() == util.Platform.WINDOWS:
-            raise SystemExit("signtool.exe is not found in %PATH%")
-        logger_server.info(
-            'signtool.exe not found, '
-            'but will not be used on this foreign platform')
-
-    code_signer = WindowsCodeSigner(codesign.config_server)
-    code_signer.run_signing_server()

build_files/buildbot/worker_bundle_dmg.py

@@ -1,551 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-import argparse
-import re
-import shutil
-import subprocess
-import sys
-import time
-
-from pathlib import Path
-from tempfile import TemporaryDirectory, NamedTemporaryFile
-from typing import List
-
-BUILDBOT_DIRECTORY = Path(__file__).absolute().parent
-CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py'
-BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent
-DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin'
-
-
-# Extra size which is added on top of actual files size when estimating size
-# of destination DNG.
-EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024
-
-################################################################################
-# Common utilities
-
-
-def get_directory_size(root_directory: Path) -> int:
-    """
-    Get size of directory on disk
-    """
-
-    total_size = 0
-    for file in root_directory.glob('**/*'):
-        total_size += file.lstat().st_size
-    return total_size
-
-
-################################################################################
-# DMG bundling specific logic
-
-def create_argument_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'source_dir',
-        type=Path,
-        help='Source directory which points to either existing .app bundle'
-             'or to a directory with .app bundles.')
-    parser.add_argument(
-        '--background-image',
-        type=Path,
-        help="Optional background picture which will be set on the DMG."
-             "If not provided default Blender's one is used.")
-    parser.add_argument(
-        '--volume-name',
-        type=str,
-        help='Optional name of a volume which will be used for DMG.')
-    parser.add_argument(
-        '--dmg',
-        type=Path,
-        help='Optional argument which points to a final DMG file name.')
-    parser.add_argument(
-        '--applescript',
-        type=Path,
-        help="Optional path to applescript to set up folder looks of DMG."
-             "If not provided default Blender's one is used.")
-    parser.add_argument(
-        '--codesign',
-        action="store_true",
-        help="Code sign and notarize DMG contents.")
-    return parser
-
-
-def collect_app_bundles(source_dir: Path) -> List[Path]:
-    """
-    Collect all app bundles which are to be put into DMG
-
-    If the source directory points to FOO.app it will be the only app bundle
-    packed.
-
-    Otherwise all .app bundles from given directory are placed to a single
-    DMG.
-    """
-
-    if source_dir.name.endswith('.app'):
-        return [source_dir]
-
-    app_bundles = []
-    for filename in source_dir.glob('*'):
-        if not filename.is_dir():
-            continue
-        if not filename.name.endswith('.app'):
-            continue
-
-        app_bundles.append(filename)
-
-    return app_bundles
-
-
-def collect_and_log_app_bundles(source_dir: Path) -> List[Path]:
-    app_bundles = collect_app_bundles(source_dir)
-
-    if not app_bundles:
-        print('No app bundles found for packing')
-        return
-
-    print(f'Found {len(app_bundles)} to pack:')
-    for app_bundle in app_bundles:
-        print(f'- {app_bundle}')
-
-    return app_bundles
-
-
-def estimate_dmg_size(app_bundles: List[Path]) -> int:
-    """
-    Estimate size of DMG to hold requested app bundles
-
-    The size is based on actual size of all files in all bundles plus some
-    space to compensate for different size-on-disk plus some space to hold
-    codesign signatures.
-
-    Is better to be on a high side since the empty space is compressed, but
-    lack of space might cause silent failures later on.
-    """
-
-    app_bundles_size = 0
-    for app_bundle in app_bundles:
-        app_bundles_size += get_directory_size(app_bundle)
-
-    return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES
-
-
-def copy_app_bundles_to_directory(app_bundles: List[Path],
-                                  directory: Path) -> None:
-    """
-    Copy all bundles to a given directory
-
-    This directory is what the DMG will be created from.
-    """
-    for app_bundle in app_bundles:
-        print(f'Copying {app_bundle.name}...')
-        shutil.copytree(app_bundle, directory / app_bundle.name)
-
-
-def get_main_app_bundle(app_bundles: List[Path]) -> Path:
-    """
-    Get application bundle main for the installation
-    """
-    return app_bundles[0]
-
-
-def create_dmg_image(app_bundles: List[Path],
-                     dmg_filepath: Path,
-                     volume_name: str) -> None:
-    """
-    Create DMG disk image and put app bundles in it
-
-    No DMG configuration or codesigning is happening here.
-    """
-
-    if dmg_filepath.exists():
-        print(f'Removing existing writable DMG {dmg_filepath}...')
-        dmg_filepath.unlink()
-
-    print('Preparing directory with app bundles for the DMG...')
-    with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str:
-        # Copy all bundles to a clean directory.
-        content_dir = Path(content_dir_str)
-        copy_app_bundles_to_directory(app_bundles, content_dir)
-
-        # Estimate size of the DMG.
-        dmg_size = estimate_dmg_size(app_bundles)
-        print(f'Estimated DMG size: {dmg_size:,} bytes.')
-
-        # Create the DMG.
-        print(f'Creating writable DMG {dmg_filepath}')
-        command = ('hdiutil',
-                   'create',
-                   '-size', str(dmg_size),
-                   '-fs', 'HFS+',
-                   '-srcfolder', content_dir,
-                   '-volname', volume_name,
-                   '-format', 'UDRW',
-                   dmg_filepath)
-        subprocess.run(command)
-
-
-def get_writable_dmg_filepath(dmg_filepath: Path):
-    """
-    Get file path for writable DMG image
-    """
-    parent = dmg_filepath.parent
-    return parent / (dmg_filepath.stem + '-temp.dmg')
-
-
-def mount_readwrite_dmg(dmg_filepath: Path) -> None:
-    """
-    Mount writable DMG
-
-    Mounting point would be /Volumes/<volume name>
-    """
-
-    print(f'Mounting read-write DMG ${dmg_filepath}')
-    command = ('hdiutil',
-               'attach', '-readwrite',
-               '-noverify',
-               '-noautoopen',
-               dmg_filepath)
-    subprocess.run(command)
-
-
-def get_mount_directory_for_volume_name(volume_name: str) -> Path:
-    """
-    Get directory under which the volume will be mounted
-    """
-
-    return Path('/Volumes') / volume_name
-
-
-def eject_volume(volume_name: str) -> None:
-    """
-    Eject given volume, if mounted
-    """
-    mount_directory = get_mount_directory_for_volume_name(volume_name)
-    if not mount_directory.exists():
-        return
-    mount_directory_str = str(mount_directory)
-
-    print(f'Ejecting volume {volume_name}')
-
-    # Figure out which device to eject.
-    mount_output = subprocess.check_output(['mount']).decode()
-    device = ''
-    for line in mount_output.splitlines():
-        if f'on {mount_directory_str} (' not in line:
-            continue
-        tokens = line.split(' ', 3)
-        if len(tokens) < 3:
-            continue
-        if tokens[1] != 'on':
-            continue
-        if device:
-            raise Exception(
-                f'Multiple devices found for mounting point {mount_directory}')
-        device = tokens[0]
-
-    if not device:
-        raise Exception(
-            f'No device found for mounting point {mount_directory}')
-
-    print(f'{mount_directory} is mounted as device {device}, ejecting...')
-    subprocess.run(['diskutil', 'eject', device])
-
-
-def copy_background_if_needed(background_image_filepath: Path,
-                              mount_directory: Path) -> None:
-    """
-    Copy background to the DMG
-
-    If the background image is not specified it will not be copied.
-    """
-
-    if not background_image_filepath:
-        print('No background image provided.')
-        return
-
-    print(f'Copying background image {background_image_filepath}')
-
-    destination_dir = mount_directory / '.background'
-    destination_dir.mkdir(exist_ok=True)
-
-    destination_filepath = destination_dir / background_image_filepath.name
-    shutil.copy(background_image_filepath, destination_filepath)
-
-
-def create_applications_link(mount_directory: Path) -> None:
-    """
-    Create link to /Applications in the given location
-    """
-
-    print('Creating link to /Applications')
-
-    command = ('ln', '-s', '/Applications', mount_directory / ' ')
-    subprocess.run(command)
-
-
-def run_applescript(applescript: Path,
-                    volume_name: str,
-                    app_bundles: List[Path],
-                    background_image_filepath: Path) -> None:
-    """
-    Run given applescript to adjust look and feel of the DMG
-    """
-
-    main_app_bundle = get_main_app_bundle(app_bundles)
-
-    with NamedTemporaryFile(
-            mode='w', suffix='.applescript') as temp_applescript:
-        print('Adjusting applescript for volume name...')
-        # Adjust script to the specific volume name.
-        with open(applescript, mode='r') as input:
-            for line in input.readlines():
-                stripped_line = line.strip()
-                if stripped_line.startswith('tell disk'):
-                    line = re.sub('tell disk ".*"',
-                                  f'tell disk "{volume_name}"',
-                                  line)
-                elif stripped_line.startswith('set background picture'):
-                    if not background_image_filepath:
-                        continue
-                    else:
-                        background_image_short = \
-                            '.background:' + background_image_filepath.name
-                        line = re.sub('to file ".*"',
-                                      f'to file "{background_image_short}"',
-                                      line)
-                line = line.replace('blender.app', main_app_bundle.name)
-                temp_applescript.write(line)
-
-        temp_applescript.flush()
-
-        print('Running applescript...')
-        command = ('osascript',  temp_applescript.name)
-        subprocess.run(command)
-
-        print('Waiting for applescript...')
-
-        # NOTE: This is copied from bundle.sh. The exact reason for sleep is
-        # still remained a mystery.
-        time.sleep(5)
-
-
-def codesign(subject: Path):
-    """
-    Codesign file or directory
-
-    NOTE: For DMG it will also notarize.
-    """
-
-    command = (CODESIGN_SCRIPT, subject)
-    subprocess.run(command)
-
-
-def codesign_app_bundles_in_dmg(mount_directory: str) -> None:
-    """
-    Code sign all binaries and bundles in the mounted directory
-    """
-
-    print(f'Codesigning all app bundles in {mount_directory}')
-    codesign(mount_directory)
-
-
-def codesign_and_notarize_dmg(dmg_filepath: Path) -> None:
-    """
-    Run codesign and notarization pipeline on the DMG
-    """
-
-    print(f'Codesigning and notarizing DMG {dmg_filepath}')
-    codesign(dmg_filepath)
-
-
-def compress_dmg(writable_dmg_filepath: Path,
-                 final_dmg_filepath: Path) -> None:
-    """
-    Compress temporary read-write DMG
-    """
-    command = ('hdiutil', 'convert',
-               writable_dmg_filepath,
-               '-format', 'UDZO',
-               '-o', final_dmg_filepath)
-
-    if final_dmg_filepath.exists():
-        print(f'Removing old compressed DMG {final_dmg_filepath}')
-        final_dmg_filepath.unlink()
-
-    print('Compressing disk image...')
-    subprocess.run(command)
-
-
-def create_final_dmg(app_bundles: List[Path],
-                     dmg_filepath: Path,
-                     background_image_filepath: Path,
-                     volume_name: str,
-                     applescript: Path,
-                     codesign: bool) -> None:
-    """
-    Create DMG with all app bundles
-
-    Will take care configuring background, signing all binaries and app bundles
-    and notarizing the DMG.
-    """
-
-    print('Running all routines to create final DMG')
-
-    writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath)
-    mount_directory = get_mount_directory_for_volume_name(volume_name)
-
-    # Make sure volume is not mounted.
-    # If it is mounted it will prevent removing old DMG files and could make
-    # it so app bundles are copied to the wrong place.
-    eject_volume(volume_name)
-
-    create_dmg_image(app_bundles, writable_dmg_filepath, volume_name)
-
-    mount_readwrite_dmg(writable_dmg_filepath)
-
-    # Run codesign first, prior to copying amything else.
-    #
-    # This allows to recurs into the content of bundles without worrying about
-    # possible interfereice of Application symlink.
-    if codesign:
-        codesign_app_bundles_in_dmg(mount_directory)
-
-    copy_background_if_needed(background_image_filepath, mount_directory)
-    create_applications_link(mount_directory)
-    run_applescript(applescript, volume_name, app_bundles,
-                    background_image_filepath)
-
-    print('Ejecting read-write DMG image...')
-    eject_volume(volume_name)
-
-    compress_dmg(writable_dmg_filepath, dmg_filepath)
-    writable_dmg_filepath.unlink()
-
-    if codesign:
-        codesign_and_notarize_dmg(dmg_filepath)
-
-
-def ensure_dmg_extension(filepath: Path) -> Path:
-    """
-    Make sure given file have .dmg extension
-    """
-
-    if filepath.suffix != '.dmg':
-        return filepath.with_suffix(f'{filepath.suffix}.dmg')
-    return filepath
-
-
-def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path:
-    """
-    Get full file path for the final DMG image
-
-    Will use the provided one when possible, otherwise will deduct it from
-    app bundles.
-
-    If the name is deducted, the DMG is stored in the current directory.
-    """
-
-    if requested_name:
-        return ensure_dmg_extension(requested_name.absolute())
-
-    # TODO(sergey): This is not necessarily the main one.
-    main_bundle = app_bundles[0]
-    # Strip .app from the name
-    return Path(main_bundle.name[:-4] + '.dmg').absolute()
-
-
-def get_background_image(requested_background_image: Path) -> Path:
-    """
-    Get effective filepath for the background image
-    """
-
-    if requested_background_image:
-        return requested_background_image.absolute()
-
-    return DARWIN_DIRECTORY / 'background.tif'
-
-
-def get_applescript(requested_applescript: Path) -> Path:
-    """
-    Get effective filepath for the applescript
-    """
-
-    if requested_applescript:
-        return requested_applescript.absolute()
-
-    return DARWIN_DIRECTORY / 'blender.applescript'
-
-
-def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str:
-    """
-    Deduct volume name from the DMG path
-
-    Will use first part of the DMG file name prior to dash.
-    """
-
-    tokens = dmg_filepath.stem.split('-')
-    words = tokens[0].split()
-
-    return ' '.join(word.capitalize() for word in words)
-
-
-def get_volume_name(requested_volume_name: str,
-                    dmg_filepath: Path) -> str:
-    """
-    Get effective name for DMG volume
-    """
-
-    if requested_volume_name:
-        return requested_volume_name
-
-    return get_volume_name_from_dmg_filepath(dmg_filepath)
-
-
-def main():
-    parser = create_argument_parser()
-    args = parser.parse_args()
-
-    # Get normalized input parameters.
-    source_dir = args.source_dir.absolute()
-    background_image_filepath = get_background_image(args.background_image)
-    applescript = get_applescript(args.applescript)
-    codesign = args.codesign
-
-    app_bundles = collect_and_log_app_bundles(source_dir)
-    if not app_bundles:
-        return
-
-    dmg_filepath = get_dmg_filepath(args.dmg, app_bundles)
-    volume_name = get_volume_name(args.volume_name, dmg_filepath)
-
-    print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)')
-
-    create_final_dmg(app_bundles,
-                     dmg_filepath,
-                     background_image_filepath,
-                     volume_name,
-                     applescript,
-                     codesign)
-
-
-if __name__ == "__main__":
-    main()

build_files/buildbot/worker_codesign.cmake

@@ -1,44 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# This is a script which is used as POST-INSTALL one for regular CMake's
-# INSTALL target.
-# It is used by buildbot workers to sign every binary which is going into
-# the final buundle.
-
-# On Windows Python 3 there only is python.exe, no python3.exe.
-#
-# On other platforms it is possible to have python2 and python3, and a
-# symbolic link to python to either of them. So on those platforms use
-# an explicit Python version.
-if(WIN32)
-  set(PYTHON_EXECUTABLE python)
-else()
-  set(PYTHON_EXECUTABLE python3)
-endif()
-
-execute_process(
-  COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/worker_codesign.py"
-          "${CMAKE_INSTALL_PREFIX}"
-  WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-  RESULT_VARIABLE exit_code
-)
-
-if(NOT exit_code EQUAL "0")
-    message(FATAL_ERROR "Non-zero exit code of codesign tool")
-endif()

build_files/buildbot/worker_codesign.py

@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# Helper script which takes care of signing provided location.
-#
-# The location can either be a directory (in which case all eligible binaries
-# will be signed) or a single file (in which case a single file will be signed).
-#
-# This script takes care of all the complexity of communicating between process
-# which requests file to be signed and the code signing server.
-#
-# NOTE: Signing happens in-place.
-
-import argparse
-import sys
-
-from pathlib import Path
-
-from codesign.simple_code_signer import SimpleCodeSigner
-
-
-def create_argument_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('path_to_sign', type=Path)
-    return parser
-
-
-def main():
-    parser = create_argument_parser()
-    args = parser.parse_args()
-    path_to_sign = args.path_to_sign.absolute()
-
-    if sys.platform == 'win32':
-        # When WIX packed is used to generate .msi on Windows the CPack will
-        # install two different projects and install them to different
-        # installation prefix:
-        #
-        # - C:\b\build\_CPack_Packages\WIX\Blender
-        # - C:\b\build\_CPack_Packages\WIX\Unspecified
-        #
-        # Annoying part is: CMake's post-install script will only be run
-        # once, with the install prefix which corresponds to a project which
-        # was installed last. But we want to sign binaries from all projects.
-        # So in order to do so we detect that we are running for a CPack's
-        # project used for WIX and force parent directory (which includes both
-        # projects) to be signed.
-        #
-        # Here we force both projects to be signed.
-        if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign):
-            path_to_sign = path_to_sign.parent
-
-    code_signer = SimpleCodeSigner()
-    code_signer.sign_file_or_directory(path_to_sign)
-
-
-if __name__ == "__main__":
-    main()

build_files/buildbot/worker_compile.py

@@ -1,135 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import os
-import shutil
-
-import buildbot_utils
-
-
-def get_cmake_options(builder):
-    codesign_script = os.path.join(
-        builder.blender_dir, 'build_files', 'buildbot', 'worker_codesign.cmake')
-
-    config_file = "build_files/cmake/config/blender_release.cmake"
-    options = ['-DCMAKE_BUILD_TYPE:STRING=Release',
-               '-DWITH_GTESTS=ON']
-
-    if builder.platform == 'mac':
-        options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
-        options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
-    elif builder.platform == 'win':
-        options.extend(['-G', 'Visual Studio 16 2019', '-A', 'x64'])
-        if builder.codesign:
-            options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + codesign_script])
-    elif builder.platform == 'linux':
-        config_file = "build_files/buildbot/config/blender_linux.cmake"
-
-    optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK-7.1')
-    options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir)
-
-    # Workaround to build sm_30 kernels with CUDA 10, since CUDA 11 no longer supports that architecture
-    if builder.platform == 'win':
-        options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1')
-        options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe')
-        options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1')
-        options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1/bin/nvcc.exe')
-    elif builder.platform == 'linux':
-        options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-10.1')
-        options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-10.1/bin/nvcc')
-        options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-11.1')
-        options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-11.1/bin/nvcc')
-
-    options.append("-C" + os.path.join(builder.blender_dir, config_file))
-    options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir))
-
-    return options
-
-
-def update_git(builder):
-    # Do extra git fetch because not all platform/git/buildbot combinations
-    # update the origin remote, causing buildinfo to detect local changes.
-    os.chdir(builder.blender_dir)
-
-    print("Fetching remotes")
-    command = ['git', 'fetch', '--all']
-    buildbot_utils.call(builder.command_prefix + command)
-
-
-def clean_directories(builder):
-    # Make sure no garbage remained from the previous run
-    if os.path.isdir(builder.install_dir):
-        shutil.rmtree(builder.install_dir)
-
-    # Make sure build directory exists and enter it
-    os.makedirs(builder.build_dir, exist_ok=True)
-
-    # Remove buildinfo files to force buildbot to re-generate them.
-    for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
-        full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo)
-        if os.path.exists(full_path):
-            print("Removing {}" . format(buildinfo))
-            os.remove(full_path)
-
-
-def cmake_configure(builder):
-    # CMake configuration
-    os.chdir(builder.build_dir)
-
-    cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt')
-    if os.path.exists(cmake_cache):
-        print("Removing CMake cache")
-        os.remove(cmake_cache)
-
-    print("CMake configure:")
-    cmake_options = get_cmake_options(builder)
-    command = ['cmake', builder.blender_dir] + cmake_options
-    buildbot_utils.call(builder.command_prefix + command)
-
-
-def cmake_build(builder):
-    # CMake build
-    os.chdir(builder.build_dir)
-
-    # NOTE: CPack will build an INSTALL target, which would mean that code
-    # signing will happen twice when using `make install` and CPack.
-    # The tricky bit here is that it is not possible to know whether INSTALL
-    # target is used by CPack or by a buildbot itaself. Extra level on top of
-    # this is that on Windows it is required to build INSTALL target in order
-    # to have unit test binaries to run.
-    # So on the one hand we do an extra unneeded code sign on Windows, but on
-    # a positive side we don't add complexity and don't make build process more
-    # fragile trying to avoid this. The signing process is way faster than just
-    # a clean build of buildbot, especially with regression tests enabled.
-    if builder.platform == 'win':
-        command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release']
-    else:
-        command = ['make', '-s', '-j16', 'install']
-
-    print("CMake build:")
-    buildbot_utils.call(builder.command_prefix + command)
-
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-    update_git(builder)
-    clean_directories(builder)
-    cmake_configure(builder)
-    cmake_build(builder)

build_files/buildbot/worker_pack.py

@@ -1,208 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Runs on buildbot worker, creating a release package using the build
-# system and zipping it into buildbot_upload.zip. This is then uploaded
-# to the master in the next buildbot step.
-
-import os
-import sys
-
-from pathlib import Path
-
-import buildbot_utils
-
-
-def get_package_name(builder, platform=None):
-    info = buildbot_utils.VersionInfo(builder)
-
-    package_name = 'blender-' + info.full_version
-    if platform:
-        package_name += '-' + platform
-    if not (builder.branch == 'master' or builder.is_release_branch):
-        if info.is_development_build:
-            package_name = builder.branch + "-" + package_name
-
-    return package_name
-
-
-def sign_file_or_directory(path):
-    from codesign.simple_code_signer import SimpleCodeSigner
-    code_signer = SimpleCodeSigner()
-    code_signer.sign_file_or_directory(Path(path))
-
-
-def create_buildbot_upload_zip(builder, package_files):
-    import zipfile
-
-    buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
-    if os.path.exists(buildbot_upload_zip):
-        os.remove(buildbot_upload_zip)
-
-    try:
-        z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
-        for filepath, filename in package_files:
-            print("Packaged", filename)
-            z.write(filepath, arcname=filename)
-        z.close()
-    except Exception as ex:
-        sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
-        sys.exit(1)
-
-
-def create_tar_xz(src, dest, package_name):
-    # One extra to remove leading os.sep when cleaning root for package_root
-    ln = len(src) + 1
-    flist = list()
-
-    # Create list of tuples containing file and archive name
-    for root, dirs, files in os.walk(src):
-        package_root = os.path.join(package_name, root[ln:])
-        flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
-
-    import tarfile
-
-    # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
-    # user compiled the package. If root then unpacks it to /usr/local/ you get
-    # a security issue.
-    def _fakeroot(tarinfo):
-        tarinfo.gid = 0
-        tarinfo.gname = "root"
-        tarinfo.uid = 0
-        tarinfo.uname = "root"
-        return tarinfo
-
-    package = tarfile.open(dest, 'w:xz', preset=9)
-    for entry in flist:
-        package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
-    package.close()
-
-
-def cleanup_files(dirpath, extension):
-    for f in os.listdir(dirpath):
-        filepath = os.path.join(dirpath, f)
-        if os.path.isfile(filepath) and f.endswith(extension):
-            os.remove(filepath)
-
-
-def pack_mac(builder):
-    info = buildbot_utils.VersionInfo(builder)
-
-    os.chdir(builder.build_dir)
-    cleanup_files(builder.build_dir, '.dmg')
-
-    package_name = get_package_name(builder, 'macOS')
-    package_filename = package_name + '.dmg'
-    package_filepath = os.path.join(builder.build_dir, package_filename)
-
-    release_dir = os.path.join(builder.blender_dir, 'release', 'darwin')
-    buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot')
-    bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py')
-
-    command = [bundle_script]
-    command += ['--dmg', package_filepath]
-    if info.is_development_build:
-        background_image = os.path.join(release_dir, 'buildbot', 'background.tif')
-        command += ['--background-image', background_image]
-    if builder.codesign:
-        command += ['--codesign']
-    command += [builder.install_dir]
-    buildbot_utils.call(command)
-
-    create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
-
-
-def pack_win(builder):
-    info = buildbot_utils.VersionInfo(builder)
-
-    os.chdir(builder.build_dir)
-    cleanup_files(builder.build_dir, '.zip')
-
-    # CPack will add the platform name
-    cpack_name = get_package_name(builder, None)
-    package_name = get_package_name(builder, 'windows' + str(builder.bits))
-
-    command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.']
-    buildbot_utils.call(builder.command_prefix + command)
-    command = ['cpack', '-G', 'ZIP']
-    buildbot_utils.call(builder.command_prefix + command)
-
-    package_filename = package_name + '.zip'
-    package_filepath = os.path.join(builder.build_dir, package_filename)
-    package_files = [(package_filepath, package_filename)]
-
-    if info.version_cycle == 'release':
-        # Installer only for final release builds, otherwise will get
-        # 'this product is already installed' messages.
-        command = ['cpack', '-G', 'WIX']
-        buildbot_utils.call(builder.command_prefix + command)
-
-        package_filename = package_name + '.msi'
-        package_filepath = os.path.join(builder.build_dir, package_filename)
-        if builder.codesign:
-            sign_file_or_directory(package_filepath)
-
-        package_files += [(package_filepath, package_filename)]
-
-    create_buildbot_upload_zip(builder, package_files)
-
-
-def pack_linux(builder):
-    blender_executable = os.path.join(builder.install_dir, 'blender')
-
-    info = buildbot_utils.VersionInfo(builder)
-
-    # Strip all unused symbols from the binaries
-    print("Stripping binaries...")
-    buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
-
-    print("Stripping python...")
-    py_target = os.path.join(builder.install_dir, info.short_version)
-    buildbot_utils.call(
-        builder.command_prefix + [
-            'find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';',
-        ],
-    )
-
-    # Construct package name
-    platform_name = 'linux64'
-    package_name = get_package_name(builder, platform_name)
-    package_filename = package_name + ".tar.xz"
-
-    print("Creating .tar.xz archive")
-    package_filepath = builder.install_dir + '.tar.xz'
-    create_tar_xz(builder.install_dir, package_filepath, package_name)
-
-    # Create buildbot_upload.zip
-    create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
-
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-
-    # Make sure install directory always exists
-    os.makedirs(builder.install_dir, exist_ok=True)
-
-    if builder.platform == 'mac':
-        pack_mac(builder)
-    elif builder.platform == 'win':
-        pack_win(builder)
-    elif builder.platform == 'linux':
-        pack_linux(builder)

build_files/buildbot/worker_test.py

@@ -1,42 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import buildbot_utils
-import os
-import sys
-
-
-def get_ctest_arguments(builder):
-    args = ['--output-on-failure']
-    if builder.platform == 'win':
-        args += ['-C', 'Release']
-    return args
-
-
-def test(builder):
-    os.chdir(builder.build_dir)
-
-    command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder)
-    buildbot_utils.call(command)
-
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-    test(builder)

build_files/buildbot/worker_update.py

@@ -1,31 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import buildbot_utils
-import os
-import sys
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-    os.chdir(builder.blender_dir)
-
-    # Run make update which handles all libraries and submodules.
-    make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py")
-    buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"])

build_files/config/README.md

@@ -0,0 +1,8 @@
+Pipeline Config
+===============
+
+This configuration file is used by buildbot new pipeline for the `update-code` step.
+
+It will soon be used by the ../utils/make_update.py script.
+
+Both buildbot and developers will eventually use the same configuration file.

build_files/config/pipeline_config.json

@@ -0,0 +1,87 @@
+{
+    "update-code":
+    {
+        "git" :
+        {
+            "submodules":
+            [
+                { "path": "release/scripts/addons", "branch": "master", "commit_id": "HEAD" },
+                { "path": "release/scripts/addons_contrib", "branch": "master", "commit_id": "HEAD" },
+                { "path": "release/datafiles/locale", "branch": "master", "commit_id": "HEAD" },
+                { "path": "source/tools", "branch": "master", "commit_id": "HEAD" }
+            ]
+        },
+        "svn": 
+        {
+            "tests": { "path": "lib/tests", "branch": "trunk", "commit_id": "HEAD" },
+            "libraries":
+            {
+                "darwin-x86_64": { "path": "lib/darwin", "branch": "trunk", "commit_id": "HEAD" },
+                "darwin-arm64": { "path": "lib/darwin_arm64", "branch": "trunk", "commit_id": "HEAD" },
+                "linux-x86_64": { "path": "lib/linux_centos7_x86_64", "branch": "trunk", "commit_id": "HEAD" },
+                "windows-amd64": { "path": "lib/win64_vc15", "branch": "trunk", "commit_id": "HEAD" }
+            }
+        }
+    },
+    "buildbot":
+    {
+        "gcc":
+        {
+            "version": "9.0"
+        },
+        "sdks":
+        {
+            "optix":
+            {
+                "version": "7.1.0"
+            },
+            "cuda10":
+            {
+                "version": "10.1"
+            },
+            "cuda11":
+            {
+                "version": "11.3"
+            }
+        },
+        "cmake":
+        {
+            "default":
+            { 
+                "version": "any",
+                "overrides":
+                {
+
+                }
+            },
+            "darwin-x86_64":
+            {
+                "overrides":
+                {
+    
+                }
+            },
+            "darwin-arm64":
+            {
+                "overrides":
+                {
+    
+                }
+            },
+            "linux-x86_64":
+            {
+                "overrides":
+                {
+    
+                }
+            },
+            "windows-amd64":
+            {
+                "overrides":
+                {
+    
+                }
+            }
+        }
+    }
+}

build_files/utils/README.md

@@ -0,0 +1,5 @@
+Make Utility Scripts
+====================
+
+Scripts used only by developers for now
+

release/darwin/README.md

@@ -0,0 +1,5 @@
+Buildbot Configuration
+======================
+
+Files used by Buildbot's `package-code-binaires` step for the darwin platform.
+

release/darwin/README.txt

@@ -1,55 +0,0 @@
-
-macOS app bundling guide
-========================
-
-Install Code Signing Certificate
---------------------------------
-
-* Go to https://developer.apple.com/account/resources/certificates/list
-* Download the Developer ID Application certificate.
-* Double click the file and add to key chain (default options).
-* Delete the file from the Downloads folder.
-
-* You will also need to install a .p12 public/private key file for the
-  certificate. This is only available for the owner of the Blender account,
-  or can be exported and copied from another system that already has code
-  signing set up.
-
-Find the codesigning identity by running:
-
-$ security find-identity -v -p codesigning
-
-"Developer ID Application: Stichting Blender Foundation" is the identity needed.
-The long code at the start of the line is used as <identity> below.
-
-Setup Apple ID
---------------
-
-* The Apple ID must have two step verification enabled.
-* Create an app specific password for the code signing app (label can be anything):
-https://support.apple.com/en-us/HT204397
-* Add the app specific password to keychain:
-
-$ security add-generic-password -a <apple-id> -w <app-specific-password> -s altool-password
-
-When running the bundle script, there will be a popup. To avoid that either:
-* Click Always Allow in the popup
-* In the Keychain Access app, change the Access Control settings on altool-password
-
-Bundle
-------
-
-Then the bundle is created as follows:
-
-$ ./bundle.sh --source <sourcedir> --dmg <dmg> --bundle-id <bundleid> --username <apple-id> --password "@keychain:altool-password" --codesign <identity>
-
-<sourcedir>  directory where built Blender.app is
-<dmg>	       location and name of the final disk image
-<bundleid>   id on notarization, for example org.blenderfoundation.blender.release
-<apple-id>   your appleid email
-<identity>   codesigning identity
-
-When specifying only --sourcedir and --dmg, the build will not be signed.
-
-Example :
-$ ./bundle.sh --source /data/build/bin --dmg /data/Blender-2.8-alpha-macOS-10.11.dmg --bundle-id org.blenderfoundation.blender.release --username "foo@mac.com" --password "@keychain:altool-password" --codesign AE825E26F12D08B692F360133210AF46F4CF7B97

release/darwin/blender.applescript

@@ -1,18 +0,0 @@
-tell application "Finder"
-          tell disk "Blender"
-               open
-               set current view of container window to icon view
-               set toolbar visible of container window to false
-               set statusbar visible of container window to false
-               set the bounds of container window to {100, 100, 640, 472}
-               set theViewOptions to icon view options of container window
-               set arrangement of theViewOptions to not arranged
-               set icon size of theViewOptions to 128
-               set background picture of theViewOptions to file ".background:background.tif"
-               set position of item " " of container window to {400, 190}
-               set position of item "blender.app" of container window to {135, 190}
-               update without registering applications
-               delay 5
-               close
-     end tell
-end tell

release/darwin/bundle.sh

@@ -1,212 +0,0 @@
-#!/usr/bin/env bash
-#
-# Script to create a macOS dmg file for Blender builds, including code
-# signing and notarization for releases.
-
-# Check that we have all needed tools.
-for i in osascript git codesign hdiutil xcrun ; do
-    if [ ! -x "$(which ${i})" ]; then
-        echo "Unable to execute command $i, macOS broken?"
-        exit 1
-    fi
-done
-
-# Defaults settings.
-_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-_volume_name="Blender"
-_tmp_dir="$(mktemp -d)"
-_tmp_dmg="/tmp/blender-tmp.dmg"
-_background_image="${_script_dir}/background.tif"
-_mount_dir="/Volumes/${_volume_name}"
-_entitlements="${_script_dir}/entitlements.plist"
-
-# Handle arguments.
-while [[ $# -gt 0 ]]; do
-    key=$1
-    case $key in
-        -s|--source)
-            SRC_DIR="$2"
-            shift
-            shift
-            ;;
-        -d|--dmg)
-            DEST_DMG="$2"
-            shift
-            shift
-            ;;
-        -b|--bundle-id)
-            N_BUNDLE_ID="$2"
-            shift
-            shift
-            ;;
-        -u|--username)
-            N_USERNAME="$2"
-            shift
-            shift
-            ;;
-        -p|--password)
-            N_PASSWORD="$2"
-            shift
-            shift
-            ;;
-        -c|--codesign)
-            C_CERT="$2"
-            shift
-            shift
-            ;;
-        --background-image)
-            _background_image="$2"
-            shift
-            shift
-            ;;
-        -h|--help)
-            echo "Usage:"
-            echo " $(basename "$0") --source DIR --dmg IMAGENAME "
-            echo "    optional arguments:"
-            echo "    --codesign <certname>"
-            echo "    --username <username>"
-            echo "    --password <password>"
-            echo "    --bundle-id <bundleid>"
-            echo " Check https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow "
-            exit 1
-            ;;
-    esac
-done
-
-if [ ! -d "${SRC_DIR}/Blender.app" ]; then
-    echo "use --source parameter to set source directory where Blender.app can be found"
-    exit 1
-fi
-
-if [ -z "${DEST_DMG}" ]; then
-    echo "use --dmg parameter to set output dmg name"
-    exit 1
-fi
-
-# Destroy destination dmg if there is any.
-test -f "${DEST_DMG}" && rm "${DEST_DMG}"
-if [ -d "${_mount_dir}" ]; then
-    echo -n "Ejecting existing blender volume.."
-    DEV_FILE=$(mount | grep "${_mount_dir}" | awk '{ print $1 }')
-    diskutil eject "${DEV_FILE}" || exit 1
-    echo
-fi
-
-# Copy dmg contents.
-echo -n "Copying Blender.app..."
-cp -r "${SRC_DIR}/Blender.app" "${_tmp_dir}/" || exit 1
-echo
-
-# Create the disk image.
-_directory_size=$(du -sh ${_tmp_dir} | awk -F'[^0-9]*' '$0=$1')
-_image_size=$(echo "${_directory_size}" + 400 | bc) # extra 400 need for codesign to work (why on earth?)
-
-echo
-echo -n "Creating disk image of size ${_image_size}M.."
-test -f "${_tmp_dmg}" && rm "${_tmp_dmg}"
-hdiutil create -size "${_image_size}m" -fs HFS+ -srcfolder "${_tmp_dir}" -volname "${_volume_name}" -format UDRW "${_tmp_dmg}" -mode 755
-
-echo "Mounting readwrite image..."
-hdiutil attach -readwrite -noverify -noautoopen "${_tmp_dmg}"
-
-echo "Setting background picture.."
-if ! test -z "${_background_image}"; then
-    echo "Copying background image ..."
-    test -d "${_mount_dir}/.background" || mkdir "${_mount_dir}/.background"
-    _background_image_NAME=$(basename "${_background_image}")
-    cp "${_background_image}" "${_mount_dir}/.background/${_background_image_NAME}"
-fi
-
-echo "Creating link to /Applications ..."
-ln -s /Applications "${_mount_dir}/Applications"
-echo "Renaming Applications to empty string."
-mv ${_mount_dir}/Applications "${_mount_dir}/ "
-
-echo "Running applescript to set folder looks ..."
-cat "${_script_dir}/blender.applescript" | osascript
-
-echo "Waiting after applescript ..."
-sleep 5
-
-if [ ! -z "${C_CERT}" ]; then
-    # Codesigning requires all libs and binaries to be signed separately.
-    echo -n "Codesigning Python"
-    for f in $(find "${_mount_dir}/Blender.app/Contents/Resources" -name "python*"); do
-        if [ -x ${f} ] && [ ! -d ${f} ]; then
-            codesign --remove-signature "${f}"
-            codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${f}"
-        fi
-    done
-    echo ; echo -n "Codesigning .dylib and .so libraries"
-    for f in $(find "${_mount_dir}/Blender.app" -name "*.dylib" -o -name "*.so"); do
-        codesign --remove-signature "${f}"
-        codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${f}"
-    done
-    echo ; echo -n "Codesigning Blender.app"
-    codesign --remove-signature "${_mount_dir}/Blender.app"
-    codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${_mount_dir}/Blender.app"
-    echo
-else
-    echo "No codesigning cert given, skipping..."
-fi
-
-# Need to eject dev files to remove /dev files and free .dmg for converting
-echo "Unmounting rw disk image ..."
-DEV_FILE=$(mount | grep "${_mount_dir}" | awk '{ print $1 }')
-diskutil eject "${DEV_FILE}"
-
-sleep 3
-
-echo "Compressing disk image ..."
-hdiutil convert "${_tmp_dmg}" -format UDZO -o "${DEST_DMG}"
-
-# Codesign the dmg
-if [ ! -z "${C_CERT}" ]; then
-    echo -n "Codesigning dmg..."
-    codesign --timestamp --force --sign "${C_CERT}" "${DEST_DMG}"
-    echo
-fi
-
-# Cleanup
-rm -rf "${_tmp_dir}"
-rm "${_tmp_dmg}"
-
-# Notarize
-if [ ! -z "${N_USERNAME}" ] && [ ! -z "${N_PASSWORD}" ] && [ ! -z "${N_BUNDLE_ID}" ]; then
-    # Send to Apple
-    echo "Sending ${DEST_DMG} for notarization..."
-    _tmpout=$(mktemp)
-    echo xcrun altool --notarize-app --verbose -f "${DEST_DMG}" --primary-bundle-id "${N_BUNDLE_ID}" --username "${N_USERNAME}" --password "${N_PASSWORD}"
-    xcrun altool --notarize-app --verbose -f "${DEST_DMG}" --primary-bundle-id "${N_BUNDLE_ID}" --username "${N_USERNAME}" --password "${N_PASSWORD}" >${_tmpout} 2>&1
-
-    # Parse request uuid
-    _requuid=$(cat "${_tmpout}" | grep "RequestUUID" | awk '{ print $3 }')
-    echo "RequestUUID: ${_requuid}"
-    if [ ! -z "${_requuid}" ]; then
-        # Wait for Apple to confirm notarization is complete
-        echo "Waiting for notarization to be complete.."
-        for c in {20..0};do
-            sleep 600
-            xcrun altool --notarization-info "${_requuid}" --username "${N_USERNAME}" --password "${N_PASSWORD}" >${_tmpout} 2>&1
-            _status=$(cat "${_tmpout}" | grep "Status:" | awk '{ print $2 }')
-            if [ "${_status}" == "invalid" ]; then
-                echo "Got invalid notarization!"
-                break;
-            fi
-
-            if [ "${_status}" == "success" ]; then
-                echo -n "Notarization successful! Stapling..."
-                xcrun stapler staple -v "${DEST_DMG}"
-                break;
-            fi
-            echo "Notarization in progress, waiting..."
-        done
-    else
-        cat ${_tmpout}
-        echo "Error getting RequestUUID, notarization unsuccessful"
-    fi
-else
-    echo "No notarization credentials supplied, skipping..."
-fi
-
-echo "..done. You should have ${DEST_DMG} ready to upload"

release/freedesktop/snap/README.md

@@ -0,0 +1,17 @@
+Snap Configuration
+===================
+
+Files used by Buildbot's `package-code-store-snap` and `deliver-code-store-snap` steps.
+
+Build pipeline snap tracks and channels
+
+```
+    <track>/stable            
+        - Latest stable release for the specified track
+    <track>/candidate         
+        - Test builds for the upcoming stable release - *not used for now*
+    <track>/beta              
+        - Nightly automated builds provided by a release branch
+    <track>/egde/<branch>
+        - Nightly or on demand builds - will also make use of branch
+```

release/freedesktop/snap/README.txt

@@ -1,38 +0,0 @@
-
-Snap Package Instructions
-=========================
-
-This folder contains the scripts for creating and uploading the snap on:
-https://snapcraft.io/blender
-
-
-Setup
------
-
-This has only been tested to work on Ubuntu.
-
-# Install required packages
-sudo apt install snapd snapcraft
-
-
-Steps
------
-
-# Build the snap file
-python3 bundle.py --version 2.XX --url https://download.blender.org/release/Blender2.XX/blender-2.XX-x86_64.tar.bz2
-
-# Install snap to test
-# --dangerous is needed since the snap has not been signed yet
-# --classic is required for installing Blender in general
-sudo snap install --dangerous --classic blender_2.XX_amd64.snap
-
-# Upload
-snapcraft push --release=stable blender_2.XX_amd64.snap
-
-
-Release Values
---------------
-
-stable: final release
-candidate: release candidates
-

release/freedesktop/snap/blender-snapcraft-template.yaml

@@ -10,12 +10,7 @@ description: |
     scientists, students, VFX experts, animators, game artists, modders, and
     the list goes on.
 
-    The standard snap channels are used in the following way:
-
-           stable - Latest stable release.
-        candidate - Test builds for the upcoming stable release.
-
-icon: ../icons/scalable/apps/blender.svg
+icon: @ICON_PATH@
 
 passthrough:
     license: GPL-3.0
@@ -27,13 +22,14 @@ apps:
         command: ./blender-wrapper
         desktop: ./blender.desktop
 
+base: core18
 version: '@VERSION@'
 grade: @GRADE@
 
 parts:
     blender:
         plugin: dump
-        source: @URL@
+        source: @PACKAGE_PATH@
         build-attributes: [keep-execstack, no-patchelf]
         override-build: |
             snapcraftctl build
@@ -47,7 +43,7 @@ parts:
             - libxrender1
             - libxxf86vm1
     wrapper:
-        plugin: copy
+        plugin: dump
         source: .
-        files:
-            blender-wrapper: blender-wrapper
+        stage:
+            - ./blender-wrapper

release/freedesktop/snap/bundle.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import pathlib
-import subprocess
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", required=True)
-parser.add_argument("--url", required=True)
-parser.add_argument("--grade", default="stable", choices=["stable", "devel"])
-args = parser.parse_args()
-
-yaml_text = pathlib.Path("snapcraft.yaml.in").read_text()
-yaml_text = yaml_text.replace("@VERSION@", args.version)
-yaml_text = yaml_text.replace("@URL@", args.url)
-yaml_text = yaml_text.replace("@GRADE@", args.grade)
-pathlib.Path("snapcraft.yaml").write_text(yaml_text)
-
-subprocess.call(["snapcraft", "clean"])
-subprocess.call(["snapcraft", "snap"])

release/steam/README.md

@@ -1,70 +0,0 @@
-Creating Steam builds for Blender
-=================================
-
-This script automates creation of the Steam files: download of the archives,
-extraction of the archives, preparation of the build scripts (VDF files), actual
-building of the Steam game files.
-
-Requirements
-============
-
-* MacOS machine - Tested on Catalina 10.15.6. Extracting contents from the DMG
-  archive did not work Windows nor on Linux using 7-zip. All DMG archives tested
-  failed to be extracted. As such only MacOS is known to work.
-* Steam SDK downloaded from SteamWorks - The `steamcmd` is used to generate the
-  Steam game files. The path to the `steamcmd` is what is actually needed.
-* SteamWorks credentials - Needed to log in using `steamcmd`.
-* Login to SteamWorks with the `steamcmd` from the command-line at least once -
-  Needded to ensure the user is properly logged in. On a new machine the user
-  will have to go through two-factor authentication.
-* App ID and Depot IDs - Needed to create the VDF files.
-* Python 3.x - 3.7 was tested.
-* Base URL - for downloading the archives.
-
-Usage
-=====
-
-```bash
-$ export STEAMUSER=SteamUserName
-$ export STEAMPW=SteamUserPW
-$ export BASEURL=https://download.blender.org/release/Blender2.83/
-$ export VERSION=2.83.3
-$ export APPID=appidnr
-$ export WINID=winidnr
-$ export LINID=linuxidnr
-$ export MACOSID=macosidnr
-
-# log in to SteamWorks from command-line at least once
-
-$ ../sdk/tools/ContentBuilder/builder_osx/steamcmd +login $STEAMUSER $STEAMPW
-
-# once that has been done we can now actually start our tool
-
-$ python3.7 create_steam_builds.py --baseurl $BASEURL --version $VERSION --appid $APPID --winid $WINID --linuxid $LINID --macosid $MACOSID --steamuser $STEAMUSER --steampw $STEAMPW --steamcmd ../sdk/tools/ContentBuilder/builder_osx/steamcmd
-```
-
-All arguments in the above example are required.
-
-At the start the tool will login using `steamcmd`. This is necessary to let the
-Steam SDK update itself if necessary.
-
-There are a few optional arguments:
-
-* `--dryrun`: If set building the game files will not actually happen. A set of
-  log files and a preview manifest per depot will be created in the output folder.
-  This can be used to double-check everything works as expected.
-* `--skipdl`: If set will skip downloading of the archives. The tool expects the
-  archives to already exist in the correct content location.
-* `--skipextract`: If set will skip extraction of all archives. The tool expects
-  the archives to already have been correctly extracted in the content location.
-
-Run the tool with `-h` for detailed information on each argument.
-
-The content and output folders are generated through appending the version
-without dots to the words `content` and `output` respectively, e.g. `content2833`
-and `output2833`. These folders are created next to the tool.
-
-From all `.template` files the Steam build scripts will be generated also in the
-same directory as the tool. The files will have the extension `.vdf`.
-
-In case of errors the tool will have a non-zero return code.

release/steam/blender_app_build.vdf.template

@@ -1,17 +0,0 @@
-"appbuild"
-{
-  "appid"  "[APPID]"
-  "desc" "Blender [VERSION]" // description for this build
-  "buildoutput" "./[OUTPUT]" // build output folder for .log, .csm & .csd files, relative to location of this file
-  "contentroot" "./[CONTENT]" // root content folder, relative to location of this file
-  "setlive"  "" // branch to set live after successful build, non if empty
-  "preview" "[DRYRUN]" // 1 to enable preview builds, 0 to commit build to steampipe
-  "local"  ""  // set to flie path of local content server
-
-  "depots"
-  {
-    "[WINID]" "depot_build_win.vdf"
-    "[LINUXID]" "depot_build_linux.vdf"
-    "[MACOSID]" "depot_build_macos.vdf"
-  }
-}

release/steam/create_steam_builds.py

@@ -1,397 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import pathlib
-import requests
-import shutil
-import subprocess
-from typing import Callable, Iterator, List, Tuple
-
-# supported archive and platform endings, used to create actual archive names
-archive_endings = ["windows64.zip", "linux64.tar.xz", "macOS.dmg"]
-
-
-def add_optional_argument(option: str, help: str) -> None:
-    global parser
-    """Add an optional argument
-
-    Args:
-        option (str): Option to add
-        help (str): Help description for the argument
-    """
-    parser.add_argument(option, help=help, action='store_const', const=1)
-
-
-def blender_archives(version: str) -> Iterator[str]:
-    """Generator for Blender archives for version.
-
-    Yields for items in archive_endings an archive name in the form of
-    blender-{version}-{ending}.
-
-    Args:
-        version (str): Version string of the form 2.83.2
-
-
-    Yields:
-        Iterator[str]: Name in the form of blender-{version}-{ending}
-    """
-    global archive_endings
-
-    for ending in archive_endings:
-        yield f"blender-{version}-{ending}"
-
-
-def get_archive_type(archive_type: str, version: str) -> str:
-    """Return the archive of given type and version.
-
-    Args:
-        archive_type (str): extension for archive type to check for
-        version (str): Version string in the form 2.83.2
-
-    Raises:
-        Exception: Execption when archive type isn't found
-
-    Returns:
-        str: archive name for given type
-    """
-
-    for archive in blender_archives(version):
-        if archive.endswith(archive_type):
-            return archive
-    raise Exception("Unknown archive type")
-
-
-def execute_command(cmd: List[str], name: str, errcode: int, cwd=".", capture_output=True) -> str:
-    """Execute the given command.
-
-    Returns the process stdout upon success if any.
-
-    On error print message the command with name that has failed. Print stdout
-    and stderr of the process if any, and then exit with given error code.
-
-    Args:
-        cmd (List[str]): Command in list format, each argument as their own item
-        name (str): Name of command to use when printing to command-line
-        errcode (int): Error code to use in case of exit()
-        cwd (str, optional): Folder to use as current work directory for command
-                             execution. Defaults to ".".
-        capture_output (bool, optional): Whether to capture command output or not.
-                                         Defaults to True.
-
-    Returns:
-        str: stdout if any, or empty string
-    """
-    cmd_process = subprocess.run(
-        cmd, capture_output=capture_output, encoding="UTF-8", cwd=cwd)
-    if cmd_process.returncode == 0:
-        if cmd_process.stdout:
-            return cmd_process.stdout
-        else:
-            return ""
-    else:
-        print(f"ERROR: {name} failed.")
-        if cmd_process.stdout:
-            print(cmd_process.stdout)
-        if cmd_process.stderr:
-            print(cmd_process.stderr)
-        exit(errcode)
-        return ""
-
-
-def download_archives(base_url: str, archives: Callable[[str], Iterator[str]], version: str, dst_dir: pathlib.Path):
-    """Download archives from the given base_url.
-
-    Archives is a generator for Blender archive names based on version.
-
-    Archive names are appended to the base_url to load from, and appended to
-    dst_dir to save to.
-
-    Args:
-        base_url (str): Base URL to load archives from
-        archives (Callable[[str], Iterator[str]]): Generator for Blender archive
-                                                   names based on version
-        version (str): Version string in the form of 2.83.2
-        dst_dir (pathlib.Path): Download destination
-    """
-
-    if base_url[-1] != '/':
-        base_url = base_url + '/'
-
-    for archive in archives(version):
-        download_url = f"{base_url}{archive}"
-        target_file = dst_dir.joinpath(archive)
-        download_file(download_url, target_file)
-
-
-def download_file(from_url: str, to_file: pathlib.Path) -> None:
-    """Download from_url as to_file.
-
-    Actual downloading will be skipped if --skipdl is given on the command-line.
-
-    Args:
-        from_url (str): Full URL to resource to download
-        to_file (pathlib.Path): Full path to save downloaded resource as
-    """
-    global args
-
-    if not args.skipdl or not to_file.exists():
-        print(f"Downloading {from_url}")
-        with open(to_file, "wb") as download_zip:
-            response = requests.get(from_url)
-            if response.status_code != requests.codes.ok:
-                print(f"ERROR: failed to download {from_url} (status code: {response.status_code})")
-                exit(1313)
-            download_zip.write(response.content)
-    else:
-        print(f"Downloading {from_url} skipped")
-    print("   ... OK")
-
-
-def copy_contents_from_dmg_to_path(dmg_file: pathlib.Path, dst: pathlib.Path) -> None:
-    """Copy the contents of the given DMG file to the destination folder.
-
-    Args:
-        dmg_file (pathlib.Path): Full path to DMG archive to extract from
-        dst (pathlib.Path): Full path to destination to extract to
-    """
-    hdiutil_attach = ["hdiutil",
-                      "attach",
-                      "-readonly",
-                      f"{dmg_file}"
-                      ]
-    attached = execute_command(hdiutil_attach, "hdiutil attach", 1)
-
-    # Last line of output is what we want, it is of the form
-    # /dev/somedisk    Apple_HFS     /Volumes/Blender
-    # We want to retain the mount point, and the folder the mount is
-    # created on. The mounted disk we need for detaching, the folder we
-    # need to be able to copy the contents to where we can use them
-    attachment_items = attached.splitlines()[-1].split()
-    mounted_disk = attachment_items[0]
-    source_location = pathlib.Path(attachment_items[2], "Blender.app")
-
-    print(f"{source_location} -> {dst}")
-
-    shutil.copytree(source_location, dst)
-
-    hdiutil_detach = ["hdiutil",
-                      "detach",
-                      f"{mounted_disk}"
-                      ]
-    execute_command(hdiutil_detach, "hdiutil detach", 2)
-
-
-def create_build_script(template_name: str, vars: List[Tuple[str, str]]) -> pathlib.Path:
-    """
-    Create the Steam build script
-
-    Use the given template and template variable tuple list.
-
-    Returns pathlib.Path to the created script.
-
-    Args:
-        template_name (str): [description]
-        vars (List[Tuple[str, str]]): [description]
-
-    Returns:
-        pathlib.Path: Full path to the generated script
-    """
-    build_script = pathlib.Path(".", template_name).read_text()
-    for var in vars:
-        build_script = build_script.replace(var[0], var[1])
-    build_script_file = template_name.replace(".template", "")
-    build_script_path = pathlib.Path(".", build_script_file)
-    build_script_path.write_text(build_script)
-    return build_script_path
-
-
-def clean_up() -> None:
-    """Remove intermediate files depending on given command-line arguments
-    """
-    global content_location, args
-
-    if not args.leavearch and not args.leaveextracted:
-        shutil.rmtree(content_location)
-
-    if args.leavearch and not args.leaveextracted:
-        shutil.rmtree(content_location.joinpath(zip_extract_folder))
-        shutil.rmtree(content_location.joinpath(tarxz_extract_folder))
-        shutil.rmtree(content_location.joinpath(dmg_extract_folder))
-
-    if args.leaveextracted and not args.leavearch:
-        import os
-        os.remove(content_location.joinpath(zipped_blender))
-        os.remove(content_location.joinpath(tarxz_blender))
-        os.remove(content_location.joinpath(dmg_blender))
-
-
-def extract_archive(archive: str, extract_folder_name: str,
-                    cmd: List[str], errcode: int) -> None:
-    """Extract all files from archive to given folder name.
-
-    Will not extract if
-    target folder already exists, or if --skipextract was given on the
-    command-line.
-
-    Args:
-        archive (str): Archive name to extract
-        extract_folder_name (str): Folder name to extract to
-        cmd (List[str]): Command with arguments to use
-        errcode (int): Error code to use for exit()
-    """
-    global args, content_location
-
-    extract_location = content_location.joinpath(extract_folder_name)
-
-    pre_extract = set(content_location.glob("*"))
-
-    if not args.skipextract or not extract_location.exists():
-        print(f"Extracting files from {archive}...")
-        cmd.append(content_location.joinpath(archive))
-        execute_command(cmd, cmd[0], errcode, cwd=content_location)
-        # in case we use a non-release archive the naming will be incorrect.
-        # simply rename to expected target name
-        post_extract = set(content_location.glob("*"))
-        diff_extract = post_extract - pre_extract
-        if not extract_location in diff_extract:
-            folder_to_rename = list(diff_extract)[0]
-            folder_to_rename.rename(extract_location)
-        print("   OK")
-    else:
-        print(f"Skipping extraction {archive}!")
-
-# ==============================================================================
-
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument("--baseurl", required=True,
-                    help="The base URL for files to download, "
-                    "i.e. https://download.blender.org/release/Blender2.83/")
-
-parser.add_argument("--version", required=True,
-                    help="The Blender version to release, in the form 2.83.3")
-
-parser.add_argument("--appid", required=True,
-                    help="The Blender App ID on Steam")
-parser.add_argument("--winid", required=True,
-                    help="The Windows depot ID")
-parser.add_argument("--linuxid", required=True,
-                    help="The Linux depot ID")
-parser.add_argument("--macosid", required=True,
-                    help="The MacOS depot ID")
-
-parser.add_argument("--steamcmd", required=True,
-                    help="Path to the steamcmd")
-parser.add_argument("--steamuser", required=True,
-                    help="The login for the Steam builder user")
-parser.add_argument("--steampw", required=True,
-                    help="Login password for the Steam builder user")
-
-add_optional_argument("--dryrun",
-                      "If set the Steam files will not be uploaded")
-add_optional_argument("--leavearch",
-                      help="If set don't clean up the downloaded archives")
-add_optional_argument("--leaveextracted",
-                      help="If set don't clean up the extraction folders")
-add_optional_argument("--skipdl",
-                      help="If set downloading the archives is skipped if it already exists locally.")
-add_optional_argument("--skipextract",
-                      help="If set skips extracting of archives. The tool assumes the archives"
-                      "have already been extracted to their correct locations")
-
-args = parser.parse_args()
-
-VERSIONNODOTS = args.version.replace('.', '')
-OUTPUT = f"output{VERSIONNODOTS}"
-CONTENT = f"content{VERSIONNODOTS}"
-
-# ===== set up main locations
-
-content_location = pathlib.Path(".", CONTENT).absolute()
-output_location = pathlib.Path(".", OUTPUT).absolute()
-
-content_location.mkdir(parents=True, exist_ok=True)
-output_location.mkdir(parents=True, exist_ok=True)
-
-# ===== login
-
-# Logging into Steam once to ensure the SDK updates itself properly. If we don't
-# do that the combined +login and +run_app_build_http at the end of the tool
-# will fail.
-steam_login = [args.steamcmd,
-               "+login",
-               args.steamuser,
-               args.steampw,
-               "+quit"
-               ]
-print("Logging in to Steam...")
-execute_command(steam_login, "Login to Steam", 10)
-print("   OK")
-
-# ===== prepare Steam build scripts
-
-template_vars = [
-    ("[APPID]", args.appid),
-    ("[OUTPUT]", OUTPUT),
-    ("[CONTENT]", CONTENT),
-    ("[VERSION]", args.version),
-    ("[WINID]", args.winid),
-    ("[LINUXID]", args.linuxid),
-    ("[MACOSID]", args.macosid),
-    ("[DRYRUN]", f"{args.dryrun}" if args.dryrun else "0")
-]
-
-blender_app_build = create_build_script(
-    "blender_app_build.vdf.template", template_vars)
-create_build_script("depot_build_win.vdf.template", template_vars)
-create_build_script("depot_build_linux.vdf.template", template_vars)
-create_build_script("depot_build_macos.vdf.template", template_vars)
-
-# ===== download archives
-
-download_archives(args.baseurl, blender_archives,
-                  args.version, content_location)
-
-# ===== set up file and folder names
-
-zipped_blender = get_archive_type("zip", args.version)
-zip_extract_folder = zipped_blender.replace(".zip", "")
-tarxz_blender = get_archive_type("tar.xz", args.version)
-tarxz_extract_folder = tarxz_blender.replace(".tar.xz", "")
-dmg_blender = get_archive_type("dmg", args.version)
-dmg_extract_folder = dmg_blender.replace(".dmg", "")
-
-# ===== extract
-
-unzip_cmd = ["unzip", "-q"]
-extract_archive(zipped_blender, zip_extract_folder,  unzip_cmd, 3)
-
-untarxz_cmd = ["tar", "-xf"]
-extract_archive(tarxz_blender, tarxz_extract_folder, untarxz_cmd, 4)
-
-if not args.skipextract or not content_location.joinpath(dmg_extract_folder).exists():
-    print("Extracting files from Blender MacOS archive...")
-    blender_dmg = content_location.joinpath(dmg_blender)
-    target_location = content_location.joinpath(
-        dmg_extract_folder, "Blender.app")
-    copy_contents_from_dmg_to_path(blender_dmg, target_location)
-    print("   OK")
-else:
-    print("Skipping extraction of .dmg!")
-
-# ===== building
-
-print("Build Steam game files...")
-steam_build = [args.steamcmd,
-               "+login",
-               args.steamuser,
-               args.steampw,
-               "+run_app_build_http",
-               blender_app_build.absolute(),
-               "+quit"
-               ]
-execute_command(steam_build, "Build with steamcmd", 13)
-print("   OK")
-
-clean_up()

release/steam/depot_build_linux.vdf.template

@@ -1,31 +0,0 @@
-"DepotBuildConfig"
-{
-  // Set your assigned depot ID here
-  "DepotID" "[LINUXID]"
-
-  // Set a root for all content.
-  // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths)
-  // will be resolved relative to this root.
-  // If you don't define ContentRoot, then it will be assumed to be
-  // the location of this script file, which probably isn't what you want
-  "ContentRoot"  "./blender-[VERSION]-linux64/"
-
-  // include all files recursivley
-  "FileMapping"
-  {
-    // This can be a full path, or a path relative to ContentRoot
-    "LocalPath" "*"
-
-    // This is a path relative to the install folder of your game
-    "DepotPath" "."
-
-    // If LocalPath contains wildcards, setting this means that all
-    // matching files within subdirectories of LocalPath will also
-    // be included.
-    "recursive" "1"
-  }
-
-  // but exclude all symbol files
-  // This can be a full path, or a path relative to ContentRoot
-  "FileExclusion" "*.pdb"
-}

release/steam/depot_build_macos.vdf.template

@@ -1,30 +0,0 @@
-"DepotBuildConfig"
-{
-  // Set your assigned depot ID here
-  "DepotID" "[MACOSID]"
-
-  // Set a root for all content.
-  // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths)
-  // will be resolved relative to this root.
-  // If you don't define ContentRoot, then it will be assumed to be
-  // the location of this script file, which probably isn't what you want
-  "ContentRoot"  "./blender-[VERSION]-macOS/"
-  // include all files recursivley
-  "FileMapping"
-  {
-    // This can be a full path, or a path relative to ContentRoot
-    "LocalPath" "*"
-
-    // This is a path relative to the install folder of your game
-    "DepotPath" "."
-
-    // If LocalPath contains wildcards, setting this means that all
-    // matching files within subdirectories of LocalPath will also
-    // be included.
-    "recursive" "1"
-  }
-
-  // but exclude all symbol files
-  // This can be a full path, or a path relative to ContentRoot
-  "FileExclusion" "*.pdb"
-}

release/steam/depot_build_win.vdf.template

@@ -1,31 +0,0 @@
-"DepotBuildConfig"
-{
-  // Set your assigned depot ID here
-  "DepotID" "[WINID]"
-
-  // Set a root for all content.
-  // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths)
-  // will be resolved relative to this root.
-  // If you don't define ContentRoot, then it will be assumed to be
-  // the location of this script file, which probably isn't what you want
-  "ContentRoot"  "./blender-[VERSION]-windows64/"
-
-  // include all files recursivley
-  "FileMapping"
-  {
-    // This can be a full path, or a path relative to ContentRoot
-    "LocalPath" "*"
-
-    // This is a path relative to the install folder of your game
-    "DepotPath" "."
-
-    // If LocalPath contains wildcards, setting this means that all
-    // matching files within subdirectories of LocalPath will also
-    // be included.
-    "recursive" "1"
-  }
-
-  // but exclude all symbol files
-  // This can be a full path, or a path relative to ContentRoot
-  "FileExclusion" "*.pdb"
-}

release/windows/msix/README.md

@@ -1,82 +1,4 @@
-create_msix_package
-===================
+Buildbot Configuration
+======================
 
-This tool is used to create MSIX packages from a given ZiP archive. The MSIX
-package is distributed mainly through the Microsoft Store. It can also be
-installed when downloaded from blender.org. For that to work the MSIX package
-needs to be signed.
-
-Requirements
-============
-
-* MakeAppX.exe - this tool is distributed with the Windows 10 SDK and is used to build the .appx package.
-* MakePri.exe - this tool is distributed with the Windows 10 SDK and is used to generate a resources file.
-* SignTool.exe - this tool is distributed with the Windows 10 SDK and is used to sign the .appx package.
-* Python 3 (3.7 or later tested) - to run the create_msix_package.py script
-* requests module - can be installed with `pip install requests`
-* PFX file (optional, but strongly recommended) - for signing the resulting MSIX
-  package. **NOTE:** If the MSIX package is not signed when uploaded to the Microsoft
-  store the validation and certification process can take up to three full
-  business day.
-
-Usage
-=====
-
-On the command-line:
-```batch
-set VERSION=2.83.4.0
-set URL=https://download.blender.org/release/Blender2.83/blender-2.83.4-windows64.zip
-set PUBID=CN=PUBIDHERE
-set PFX=X:\path\to\cert.pfx
-set PFXPW=pwhere
-
-python create_msix_package.py --version %VERSION% --url %URL% --publisher %PUBID% --pfx %PFX% --password %PFXPW%
-```
-
-Result will be a MSIX package with the name `blender-2.83.4-windows64.msix`.
-With the above usage it will be signed. If the signing options are left out the
-package will not be signed.
-
-Optional arguments
-==================
-
-In support of testing and developing the manifest and scripts there are a few
-optional arguments:
-
-* `--skipdl` : If a `blender.zip` is available already next to the tool use this
-  to skip actual downloading of the archive designated by `--url`. The latter
-  option is still required
-* `--overwrite` : When script fails the final clean-up may be incomplete leaving
-  the `Content` folder with its structure. Specify this argument to automatically
-  clean up this folder before starting to seed the `Content` folder
-* `--leavezip` : When specified leave the `blender.zip` file while cleaning up
-  all other intermediate files, including the `Content` folder. This is useful
-  to not have to re-download the same archive from `--url` on each usage
-
-
-What it does
-============
-
-The tool creates in the directory it lives a subfolder called `Content`. This is
-where all necessary files are placed.
-
-The `Assets` folder is copied to the `Content` folder.
-
-From the application manifest template a version with necessary parts replaced as
-their actual values as specified on the command-line is realized. This manifest controls the packaging of Blender into the MSIX format.
-
-Next the tool downloads the designated ZIP archive locally as blender.zip. From
-this archive the files are extracted into the `Content\Blender` folder, but skip
-the leading part of paths in the ZIP. We want to write the files to the
-content_blender_folder where blender.exe ends up as
-`Content\Blender\blender.exe`, and not
-`Content\Blender\blender-2.83.4-windows64\blender.exe`
-
-Once the extraction is completed the MakeAppX tool is executed with the `Content`
-folder as input. The result will be the MSIX package with the name in the form
-`blender-X.YY.Z-windows64.msix`.
-
-If the PFX file and its password are given on the command-line this MSIX package
-will be signed.
-
-All intermediate files and directories will be removed.
+Files used by Buildbot's `package-code-store-windows` step.

release/windows/msix/create_msix_package.py

@@ -1,197 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import pathlib
-import requests
-import shutil
-import subprocess
-import zipfile
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--version",
-    required=True,
-    help="Version string in the form of 2.83.3.0",
-)
-parser.add_argument(
-    "--url",
-    required=True,
-    help="Location of the release ZIP archive to download",
-)
-parser.add_argument(
-    "--publisher",
-    required=True,
-    help="A string in the form of 'CN=PUBLISHER'",
-)
-parser.add_argument(
-    "--pfx",
-    required=False,
-    help="Absolute path to the PFX file used for signing the resulting MSIX package",
-)
-parser.add_argument(
-    "--password",
-    required=False,
-    default="blender",
-    help="Password for the PFX file",
-)
-parser.add_argument(
-    "--lts",
-    required=False,
-    help="If set this MSIX is for an LTS release",
-    action='store_const',
-    const=1,
-)
-parser.add_argument(
-    "--skipdl",
-    required=False,
-    help="If set skip downloading of the specified URL as blender.zip. The tool assumes blender.zip exists",
-    action='store_const',
-    const=1,
-)
-parser.add_argument(
-    "--leavezip",
-    required=False,
-    help="If set don't clean up the downloaded blender.zip",
-    action='store_const',
-    const=1,
-)
-parser.add_argument(
-    "--overwrite",
-    required=False,
-    help="If set remove Content folder if it already exists",
-    action='store_const',
-    const=1,
-)
-args = parser.parse_args()
-
-
-def execute_command(cmd: list, name: str, errcode: int):
-    """
-    Execute given command in cmd. Output is captured. If an error
-    occurs name is used to print ERROR message, along with stderr and
-    stdout of the process if either was captured.
-    """
-    cmd_process = subprocess.run(cmd, capture_output=True, encoding="UTF-8")
-    if cmd_process.returncode != 0:
-        print(f"ERROR: {name} failed.")
-        if cmd_process.stdout:
-            print(cmd_process.stdout)
-        if cmd_process.stderr:
-            print(cmd_process.stderr)
-        exit(errcode)
-
-
-LTSORNOT = ""
-PACKAGETYPE = ""
-if args.lts:
-    versionparts = args.version.split(".")
-    LTSORNOT = f" {versionparts[0]}.{versionparts[1]} LTS"
-    PACKAGETYPE = f"{versionparts[0]}.{versionparts[1]}LTS"
-
-blender_package_msix = pathlib.Path(".", f"blender-{args.version}-windows64.msix").absolute()
-content_folder = pathlib.Path(".", "Content")
-content_blender_folder = pathlib.Path(content_folder, "Blender").absolute()
-content_assets_folder = pathlib.Path(content_folder, "Assets")
-assets_original_folder = pathlib.Path(".", "Assets")
-
-pri_config_file = pathlib.Path(".", "priconfig.xml")
-pri_resources_file = pathlib.Path(content_folder, "resources.pri")
-
-local_blender_zip = pathlib.Path(".", "blender.zip")
-
-if args.pfx:
-    pfx_path = pathlib.Path(args.pfx)
-    if not pfx_path.exists():
-        print("ERROR: PFX file not found. Please ensure you give the correct path to the PFX file on the command-line.")
-        exit(1)
-    print(f"Creating MSIX package with signing using PFX file at {pfx_path}")
-else:
-    pfx_path = None
-    print("Creating MSIX package without signing.")
-
-pri_command = ["makepri",
-                   "new",
-                   "/pr", f"{content_folder.absolute()}",
-                   "/cf", f"{pri_config_file.absolute()}",
-                   "/of", f"{pri_resources_file.absolute()}"
-                   ]
-
-msix_command = ["makeappx",
-                "pack",
-                "/h", "SHA256",
-                "/d", f"{content_folder.absolute()}",
-                "/p", f"{blender_package_msix}"
-                ]
-if pfx_path:
-    sign_command = ["signtool",
-                    "sign",
-                    "/fd", "sha256",
-                    "/a", "/f", f"{pfx_path.absolute()}",
-                    "/p", f"{args.password}",
-                    f"{blender_package_msix}"
-                    ]
-
-if args.overwrite:
-    if content_folder.joinpath("Assets").exists():
-        shutil.rmtree(content_folder)
-content_folder.mkdir(exist_ok=True)
-shutil.copytree(assets_original_folder, content_assets_folder)
-
-manifest_text = pathlib.Path("AppxManifest.xml.template").read_text()
-manifest_text = manifest_text.replace("[VERSION]", args.version)
-manifest_text = manifest_text.replace("[PUBLISHER]", args.publisher)
-manifest_text = manifest_text.replace("[LTSORNOT]", LTSORNOT)
-manifest_text = manifest_text.replace("[PACKAGETYPE]", PACKAGETYPE)
-pathlib.Path(content_folder, "AppxManifest.xml").write_text(manifest_text)
-
-if not args.skipdl:
-    print(f"Downloading blender archive {args.url} to {local_blender_zip}...")
-
-    with open(local_blender_zip, "wb") as download_zip:
-        response = requests.get(args.url)
-        download_zip.write(response.content)
-
-    print("... download complete.")
-else:
-    print("Skipping download")
-
-print(f"Extracting files from ZIP to {content_blender_folder}...")
-
-# Extract the files from the ZIP archive, but skip the leading part of paths
-# in the ZIP. We want to write the files to the content_blender_folder where
-# blender.exe ends up as ./Content/Blender/blender.exe, and not
-# ./Content/Blender/blender-2.83.3-windows64/blender.exe
-with zipfile.ZipFile(local_blender_zip, "r") as blender_zip:
-    for entry in blender_zip.infolist():
-        if entry.is_dir():
-            continue
-        entry_location = pathlib.Path(entry.filename)
-        target_location = content_blender_folder.joinpath(*entry_location.parts[1:])
-        pathlib.Path(target_location.parent).mkdir(parents=True, exist_ok=True)
-        extracted_entry = blender_zip.read(entry)
-        target_location.write_bytes(extracted_entry)
-
-print("... extraction complete.")
-
-
-print(f"Generating Package Resource Index (PRI) file using command: {' '.join(pri_command)}")
-execute_command(pri_command, "MakePri", 4)
-
-print(f"Creating MSIX package using command: {' '.join(msix_command)}")
-
-# Remove MSIX file if it already exists. Otherwise the MakeAppX tool
-# will hang.
-if blender_package_msix.exists():
-    os.remove(blender_package_msix)
-execute_command(msix_command, "MakeAppX", 2)
-
-if args.pfx:
-    print(f"Signing MSIX package using command: {' '.join(sign_command)}")
-    execute_command(sign_command, "SignTool", 3)
-
-if not args.leavezip:
-    os.remove(local_blender_zip)
-shutil.rmtree(content_folder)
-
-print("Done.")