tag active viewer

Differential Revision: https://developer.blender.org/D11462

build_files/build_environment/cmake/blosc.cmake

@@ -29,7 +29,7 @@ set(BLOSC_EXTRA_ARGS
   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
 )
 
-# Prevent blosc from including it's own local copy of zlib in the object file
+# Prevent blosc from including its own local copy of zlib in the object file
 # and cause linker errors with everybody else.
 set(BLOSC_EXTRA_ARGS ${BLOSC_EXTRA_ARGS}
   -DPREFER_EXTERNAL_ZLIB=ON

build_files/build_environment/cmake/versions.cmake

@@ -43,7 +43,7 @@ set(JPEG_FILE libjpeg-turbo-${JPEG_VERSION}.tar.gz)
 set(BOOST_VERSION 1.73.0)
 set(BOOST_VERSION_NODOTS 1_73_0)
 set(BOOST_VERSION_NODOTS_SHORT 1_73)
-set(BOOST_URI https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_NODOTS}.tar.gz)
+set(BOOST_URI https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_NODOTS}.tar.gz)
 set(BOOST_HASH 4036cd27ef7548b8d29c30ea10956196)
 set(BOOST_HASH_TYPE MD5)
 set(BOOST_FILE boost_${BOOST_VERSION_NODOTS}.tar.gz)
@@ -297,10 +297,10 @@ set(OPENJPEG_HASH 63f5a4713ecafc86de51bfad89cc07bb788e9bba24ebbf0c4ca637621aadb6
 set(OPENJPEG_HASH_TYPE SHA256)
 set(OPENJPEG_FILE openjpeg-v${OPENJPEG_VERSION}.tar.gz)
 
-set(FFMPEG_VERSION 4.2.3)
+set(FFMPEG_VERSION 4.4)
 set(FFMPEG_URI http://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2)
-set(FFMPEG_HASH 695fad11f3baf27784e24cb0e977b65a)
-set(FFMPEG_HASH_TYPE MD5)
+set(FFMPEG_HASH 42093549751b582cf0f338a21a3664f52e0a9fbe0d238d3c992005e493607d0e)
+set(FFMPEG_HASH_TYPE SHA256)
 set(FFMPEG_FILE ffmpeg-${FFMPEG_VERSION}.tar.bz2)
 
 set(FFTW_VERSION 3.3.8)

build_files/build_environment/install_deps.sh

@@ -37,7 +37,7 @@ if [ $USE_DEBUG_TRAP -ne 0 ]; then
   trap 'err_report $LINENO' ERR
 fi
 
-# Noisy, show every line that runs with it's line number.
+# Noisy, show every line that runs with its line number.
 if [ $USE_DEBUG_LOG -ne 0 ]; then
   PS4='\e[0;33m$(printf %4d ${LINENO}):\e\033[0m '
   set -x
@@ -563,9 +563,9 @@ OIDN_SKIP=false
 
 ISPC_VERSION="1.14.1"
 
-FFMPEG_VERSION="4.2.3"
-FFMPEG_VERSION_SHORT="4.2"
-FFMPEG_VERSION_MIN="3.0"
+FFMPEG_VERSION="4.4"
+FFMPEG_VERSION_SHORT="4.4"
+FFMPEG_VERSION_MIN="4.4"
 FFMPEG_VERSION_MAX="5.0"
 FFMPEG_FORCE_BUILD=false
 FFMPEG_FORCE_REBUILD=false

build_files/build_environment/patches/cmake/modules/FindIlmBase.cmake

@@ -20,7 +20,7 @@
 # ILMBASE_LIBRARIES   - list of libraries to link against when using IlmBase.
 # ILMBASE_FOUND       - True if IlmBase was found.
 
-# Other standarnd issue macros
+# Other standard issue macros
 include(FindPackageHandleStandardArgs)
 include(FindPackageMessage)
 include(SelectLibraryConfigurations)

build_files/build_environment/patches/cmake/modules/FindOpenEXR.cmake

@@ -22,7 +22,7 @@
 #                       These are defined by the FindIlmBase module.
 # OPENEXR_FOUND       - True if OpenEXR was found.
 
-# Other standarnd issue macros
+# Other standard issue macros
 include(SelectLibraryConfigurations)
 include(FindPackageHandleStandardArgs)
 include(FindPackageMessage)

build_files/buildbot/README.md

@@ -1,70 +1,4 @@
-Blender Buildbot
-================
+Buildbot Configuration
+=====================
 
-Code signing
-------------
-
-Code signing is done as part of INSTALL target, which makes it possible to sign
-files which are aimed into a bundle and coming from a non-signed source (such as
-libraries SVN).
-
-This is achieved by specifying `worker_codesign.cmake` as a post-install script
-run by CMake. This CMake script simply involves an utility script written in
-Python which takes care of an actual signing.
-
-### Configuration
-
-Client configuration doesn't need anything special, other than variable
-`SHARED_STORAGE_DIR` pointing to a location which is watched by a server.
-This is done in `config_builder.py` file and is stored in Git (which makes it
-possible to have almost zero-configuration buildbot machines).
-
-Server configuration requires copying `config_server_template.py` under the
-name of `config_server.py` and tweaking values, which are platform-specific.
-
-#### Windows configuration
-
-There are two things which are needed on Windows in order to have code signing
-to work:
-
-- `TIMESTAMP_AUTHORITY_URL` which is most likely set http://timestamp.digicert.com
-- `CERTIFICATE_FILEPATH` which is a full file path to a PKCS #12 key (.pfx).
-
-## Tips
-
-### Self-signed certificate on Windows
-
-It is easiest to test configuration using self-signed certificate.
-
-The certificate manipulation utilities are coming with Windows SDK.
-Unfortunately, they are not added to PATH. Here is an example of how to make
-sure they are easily available:
-
-```
-set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH%
-set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64;%PATH%
-```
-
-Generate CA:
-
-```
-makecert -r -pe -n "CN=Blender Test CA" -ss CA -sr CurrentUser -a sha256 ^
-         -cy authority -sky signature -sv BlenderTestCA.pvk BlenderTestCA.cer
-```
-
-Import the generated CA:
-
-```
-certutil -user -addstore Root BlenderTestCA.cer
-```
-
-Create self-signed certificate and pack it into PKCS #12:
-
-```
-makecert -pe -n "CN=Blender Test SPC" -a sha256 -cy end ^
-         -sky signature ^
-         -ic BlenderTestCA.cer -iv BlenderTestCA.pvk ^
-         -sv BlenderTestSPC.pvk BlenderTestSPC.cer
-
-pvk2pfx -pvk BlenderTestSPC.pvk -spc BlenderTestSPC.cer -pfx BlenderTestSPC.pfx
-```
+Files used by Buildbot's `compile-code` step.

build_files/buildbot/buildbot_utils.py

@@ -1,127 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import argparse
-import os
-import re
-import subprocess
-import sys
-
-
-def is_tool(name):
-    """Check whether `name` is on PATH and marked as executable."""
-
-    # from whichcraft import which
-    from shutil import which
-
-    return which(name) is not None
-
-
-class Builder:
-    def __init__(self, name, branch, codesign):
-        self.name = name
-        self.branch = branch
-        self.is_release_branch = re.match("^blender-v(.*)-release$", branch) is not None
-        self.codesign = codesign
-
-        # Buildbot runs from build/ directory
-        self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git'))
-        self.build_dir = os.path.abspath(os.path.join('..', 'build'))
-        self.install_dir = os.path.abspath(os.path.join('..', 'install'))
-        self.upload_dir = os.path.abspath(os.path.join('..', 'install'))
-
-        # Detect platform
-        if name.startswith('mac'):
-            self.platform = 'mac'
-            self.command_prefix = []
-        elif name.startswith('linux'):
-            self.platform = 'linux'
-            if is_tool('scl'):
-                self.command_prefix = ['scl', 'enable', 'devtoolset-9', '--']
-            else:
-                self.command_prefix = []
-        elif name.startswith('win'):
-            self.platform = 'win'
-            self.command_prefix = []
-        else:
-            raise ValueError('Unkonw platform for builder ' + self.platform)
-
-        # Always 64 bit now
-        self.bits = 64
-
-
-def create_builder_from_arguments():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('builder_name')
-    parser.add_argument('branch', default='master', nargs='?')
-    parser.add_argument("--codesign", action="store_true")
-    args = parser.parse_args()
-    return Builder(args.builder_name, args.branch, args.codesign)
-
-
-class VersionInfo:
-    def __init__(self, builder):
-        # Get version information
-        buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h")
-        blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
-
-        version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION'))
-        version_number_patch = int(self._parse_header_file(blender_h, 'BLENDER_VERSION_PATCH'))
-        version_numbers = (version_number // 100, version_number % 100, version_number_patch)
-        self.short_version = "%d.%d" % (version_numbers[0], version_numbers[1])
-        self.version = "%d.%d.%d" % version_numbers
-        self.version_cycle = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE')
-        self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
-
-        if self.version_cycle == "release":
-            # Final release
-            self.full_version = self.version
-            self.is_development_build = False
-        elif self.version_cycle == "rc":
-            # Release candidate
-            self.full_version = self.version + self.version_cycle
-            self.is_development_build = False
-        else:
-            # Development build
-            self.full_version = self.version + '-' + self.hash
-            self.is_development_build = True
-
-    def _parse_header_file(self, filename, define):
-        import re
-        regex = re.compile(r"^#\s*define\s+%s\s+(.*)" % define)
-        with open(filename, "r") as file:
-            for l in file:
-                match = regex.match(l)
-                if match:
-                    return match.group(1)
-        return None
-
-
-def call(cmd, env=None, exit_on_error=True):
-    print(' '.join(cmd))
-
-    # Flush to ensure correct order output on Windows.
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-    retcode = subprocess.call(cmd, env=env)
-    if exit_on_error and retcode != 0:
-        sys.exit(retcode)
-    return retcode

build_files/buildbot/codesign/absolute_and_relative_filename.py

@@ -1,81 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import List
-
-
-@dataclass
-class AbsoluteAndRelativeFileName:
-    """
-    Helper class which keeps track of absolute file path for a direct access and
-    corresponding relative path against given base.
-
-    The relative part is used to construct a file name within an archive which
-    contains files which are to be signed or which has been signed already
-    (depending on whether the archive is addressed to signing server or back
-    to the buildbot worker).
-    """
-
-    # Base directory which is where relative_filepath is relative to.
-    base_dir: Path
-
-    # Full absolute path of the corresponding file.
-    absolute_filepath: Path
-
-    # Derived from full file path, contains part of the path which is relative
-    # to a desired base path.
-    relative_filepath: Path
-
-    def __init__(self, base_dir: Path, filepath: Path):
-        self.base_dir = base_dir
-        self.absolute_filepath = filepath.resolve()
-        self.relative_filepath = self.absolute_filepath.relative_to(
-            self.base_dir)
-
-    @classmethod
-    def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName':
-        assert path.is_absolute()
-        assert path.is_file()
-
-        base_dir = path.parent
-        return AbsoluteAndRelativeFileName(base_dir, path)
-
-    @classmethod
-    def recursively_from_directory(cls, base_dir: Path) \
-            -> List['AbsoluteAndRelativeFileName']:
-        """
-        Create list of AbsoluteAndRelativeFileName for all the files in the
-        given directory.
-
-        NOTE: Result will be pointing to a resolved paths.
-        """
-        assert base_dir.is_absolute()
-        assert base_dir.is_dir()
-
-        base_dir = base_dir.resolve()
-
-        result = []
-        for filename in base_dir.glob('**/*'):
-            if not filename.is_file():
-                continue
-            result.append(AbsoluteAndRelativeFileName(base_dir, filename))
-        return result

build_files/buildbot/codesign/archive_with_indicator.py

@@ -1,245 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import dataclasses
-import json
-import os
-
-from pathlib import Path
-from typing import Optional
-
-import codesign.util as util
-
-
-class ArchiveStateError(Exception):
-    message: str
-
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)
-
-
-@dataclasses.dataclass
-class ArchiveState:
-    """
-    Additional information (state) of the archive
-
-    Includes information like expected file size of the archive file in the case
-    the archive file is expected to be successfully created.
-
-    If the archive can not be created, this state will contain error message
-    indicating details of error.
-    """
-
-    # Size in bytes of the corresponding archive.
-    file_size: Optional[int] = None
-
-    # Non-empty value indicates that error has happenned.
-    error_message: str = ''
-
-    def has_error(self) -> bool:
-        """
-        Check whether the archive is at error state
-        """
-
-        return self.error_message
-
-    def serialize_to_string(self) -> str:
-        payload = dataclasses.asdict(self)
-        return json.dumps(payload, sort_keys=True, indent=4)
-
-    def serialize_to_file(self, filepath: Path) -> None:
-        string = self.serialize_to_string()
-        filepath.write_text(string)
-
-    @classmethod
-    def deserialize_from_string(cls, string: str) -> 'ArchiveState':
-        try:
-            object_as_dict = json.loads(string)
-        except json.decoder.JSONDecodeError:
-            raise ArchiveStateError('Error parsing JSON')
-
-        return cls(**object_as_dict)
-
-    @classmethod
-    def deserialize_from_file(cls, filepath: Path):
-        string = filepath.read_text()
-        return cls.deserialize_from_string(string)
-
-
-class ArchiveWithIndicator:
-    """
-    The idea of this class is to wrap around logic which takes care of keeping
-    track of a name of an archive and synchronization routines between buildbot
-    worker and signing server.
-
-    The synchronization is done based on creating a special file after the
-    archive file is knowingly ready for access.
-    """
-
-    # Base directory where the archive is stored (basically, a basename() of
-    # the absolute archive file name).
-    #
-    # For example, 'X:\\TEMP\\'.
-    base_dir: Path
-
-    # Absolute file name of the archive.
-    #
-    # For example, 'X:\\TEMP\\FOO.ZIP'.
-    archive_filepath: Path
-
-    # Absolute name of a file which acts as an indication of the fact that the
-    # archive is ready and is available for access.
-    #
-    # This is how synchronization between buildbot worker and signing server is
-    # done:
-    # - First, the archive is created under archive_filepath name.
-    # - Second, the indication file is created under ready_indicator_filepath
-    #   name.
-    # - Third, the colleague of whoever created the indicator name watches for
-    #   the indication file to appear, and once it's there it access the
-    #   archive.
-    ready_indicator_filepath: Path
-
-    def __init__(
-            self, base_dir: Path, archive_name: str, ready_indicator_name: str):
-        """
-        Construct the object from given base directory and name of the archive
-        file:
-          ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY')
-        """
-
-        self.base_dir = base_dir
-        self.archive_filepath = self.base_dir / archive_name
-        self.ready_indicator_filepath = self.base_dir / ready_indicator_name
-
-    def is_ready_unsafe(self) -> bool:
-        """
-        Check whether the archive is ready for access.
-
-        No guarding about possible network failres is done here.
-        """
-        if not self.ready_indicator_filepath.exists():
-            return False
-
-        try:
-            archive_state = ArchiveState.deserialize_from_file(
-                self.ready_indicator_filepath)
-        except ArchiveStateError as error:
-            print(f'Error deserializing archive state: {error.message}')
-            return False
-
-        if archive_state.has_error():
-            # If the error did happen during codesign procedure there will be no
-            # corresponding archive file.
-            # The caller code will deal with the error check further.
-            return True
-
-        # Sometimes on macOS indicator file appears prior to the actual archive
-        # despite the order of creation and os.sync() used in tag_ready().
-        # So consider archive not ready if there is an indicator without an
-        # actual archive.
-        if not self.archive_filepath.exists():
-            print('Found indicator without actual archive, waiting for archive '
-                  f'({self.archive_filepath}) to appear.')
-            return False
-
-        # Wait for until archive is fully stored.
-        actual_archive_size = self.archive_filepath.stat().st_size
-        if actual_archive_size != archive_state.file_size:
-            print('Partial/invalid archive size (expected '
-                  f'{archive_state.file_size} got {actual_archive_size})')
-            return False
-
-        return True
-
-    def is_ready(self) -> bool:
-        """
-        Check whether the archive is ready for access.
-
-        Will tolerate possible network failures: if there is a network failure
-        or if there is still no proper permission on a file False is returned.
-        """
-
-        # There are some intermitten problem happening at a random which is
-        # translates to "OSError : [WinError 59] An unexpected network error occurred".
-        # Some reports suggests it might be due to lack of permissions to the file,
-        # which might be applicable in our case since it's possible that file is
-        # initially created with non-accessible permissions and gets chmod-ed
-        # after initial creation.
-        try:
-            return self.is_ready_unsafe()
-        except OSError as e:
-            print(f'Exception checking archive: {e}')
-            return False
-
-    def tag_ready(self, error_message='') -> None:
-        """
-        Tag the archive as ready by creating the corresponding indication file.
-
-        NOTE: It is expected that the archive was never tagged as ready before
-              and that there are no subsequent tags of the same archive.
-              If it is violated, an assert will fail.
-        """
-        assert not self.is_ready()
-
-        # Try the best to make sure everything is synced to the file system,
-        # to avoid any possibility of stamp appearing on a network share prior to
-        # an actual file.
-        if util.get_current_platform() != util.Platform.WINDOWS:
-            os.sync()
-
-        archive_size = -1
-        if self.archive_filepath.exists():
-            archive_size = self.archive_filepath.stat().st_size
-
-        archive_info = ArchiveState(
-            file_size=archive_size, error_message=error_message)
-
-        self.ready_indicator_filepath.write_text(
-            archive_info.serialize_to_string())
-
-    def get_state(self) -> ArchiveState:
-        """
-        Get state object for this archive
-
-        The state is read from the corresponding state file.
-        """
-
-        try:
-            return ArchiveState.deserialize_from_file(self.ready_indicator_filepath)
-        except ArchiveStateError as error:
-            return ArchiveState(error_message=f'Error in information format: {error}')
-
-    def clean(self) -> None:
-        """
-        Remove both archive and the ready indication file.
-        """
-        util.ensure_file_does_not_exist_or_die(self.ready_indicator_filepath)
-        util.ensure_file_does_not_exist_or_die(self.archive_filepath)
-
-    def is_fully_absent(self) -> bool:
-        """
-        Check whether both archive and its ready indicator are absent.
-        Is used for a sanity check during code signing process by both
-        buildbot worker and signing server.
-        """
-        return (not self.archive_filepath.exists() and
-                not self.ready_indicator_filepath.exists())

build_files/buildbot/codesign/base_code_signer.py

@@ -1,501 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Signing process overview.
-#
-# From buildbot worker side:
-#  - Files which needs to be signed are collected from either a directory to
-#    sign all signable files in there, or by filename of a single file to sign.
-#  - Those files gets packed into an archive and stored in a location location
-#    which is watched by the signing server.
-#  - A marker READY file is created which indicates the archive is ready for
-#    access.
-#  - Wait for the server to provide an archive with signed files.
-#    This is done by watching for the READY file which corresponds to an archive
-#    coming from the signing server.
-#  - Unpack the signed signed files from the archives and replace original ones.
-#
-# From code sign server:
-#  - Watch special location for a READY file which indicates the there is an
-#    archive with files which are to be signed.
-#  - Unpack the archive to a temporary location.
-#  - Run codesign tool and make sure all the files are signed.
-#  - Pack the signed files and store them in a location which is watched by
-#    the buildbot worker.
-#  - Create a READY file which indicates that the archive with signed files is
-#    ready.
-
-import abc
-import logging
-import shutil
-import subprocess
-import time
-import tarfile
-import uuid
-
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import Iterable, List
-
-import codesign.util as util
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.archive_with_indicator import ArchiveWithIndicator
-from codesign.exception import CodeSignException
-
-
-logger = logging.getLogger(__name__)
-logger_builder = logger.getChild('builder')
-logger_server = logger.getChild('server')
-
-
-def pack_files(files: Iterable[AbsoluteAndRelativeFileName],
-               archive_filepath: Path) -> None:
-    """
-    Create tar archive from given files for the signing pipeline.
-    Is used by buildbot worker to create an archive of files which are to be
-    signed, and by signing server to send signed files back to the worker.
-    """
-    with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle:
-        for file_info in files:
-            tar_file_handle.add(file_info.absolute_filepath,
-                                arcname=file_info.relative_filepath)
-
-
-def extract_files(archive_filepath: Path,
-                  extraction_dir: Path) -> None:
-    """
-    Extract all files form the given archive into the given direcotry.
-    """
-
-    # TODO(sergey): Verify files in the archive have relative path.
-
-    with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle:
-        tar_file_handle.extractall(path=extraction_dir)
-
-
-class BaseCodeSigner(metaclass=abc.ABCMeta):
-    """
-    Base class for a platform-specific signer of binaries.
-
-    Contains all the logic shared across platform-specific implementations, such
-    as synchronization and notification logic.
-
-    Platform specific bits (such as actual command for signing the binary) are
-    to be implemented as a subclass.
-
-    Provides utilities code signing as a whole, including functionality needed
-    by a signing server and a buildbot worker.
-
-    The signer and builder may run on separate machines, the only requirement is
-    that they have access to a directory which is shared between them. For the
-    security concerns this is to be done as a separate machine (or as a Shared
-    Folder configuration in VirtualBox configuration). This directory might be
-    mounted under different base paths, but its underlying storage is to be
-    the same.
-
-    The code signer is short-lived on a buildbot worker side, and is living
-    forever on a code signing server side.
-    """
-
-    # TODO(sergey): Find a neat way to have config annotated.
-    # config: Config
-
-    # Storage directory where builder puts files which are requested to be
-    # signed.
-    # Consider this an input of the code signing server.
-    unsigned_storage_dir: Path
-
-    # Storage where signed files are stored.
-    # Consider this an output of the code signer server.
-    signed_storage_dir: Path
-
-    # Platform the code is currently executing on.
-    platform: util.Platform
-
-    def __init__(self, config):
-        self.config = config
-
-        absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve()
-
-        # Unsigned (signing server input) configuration.
-        self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned'
-
-        # Signed (signing server output) configuration.
-        self.signed_storage_dir = absolute_shared_storage_dir / 'signed'
-
-        self.platform = util.get_current_platform()
-
-    def cleanup_environment_for_builder(self) -> None:
-        # TODO(sergey): Revisit need of cleaning up the existing files.
-        # In practice it wasn't so helpful, and with multiple clients
-        # talking to the same server it becomes even more tricky.
-        pass
-
-    def cleanup_environment_for_signing_server(self) -> None:
-        # TODO(sergey): Revisit need of cleaning up the existing files.
-        # In practice it wasn't so helpful, and with multiple clients
-        # talking to the same server it becomes even more tricky.
-        pass
-
-    def generate_request_id(self) -> str:
-        """
-        Generate an unique identifier for code signing request.
-        """
-        return str(uuid.uuid4())
-
-    def archive_info_for_request_id(
-            self, path: Path, request_id: str) -> ArchiveWithIndicator:
-        return ArchiveWithIndicator(
-            path, f'{request_id}.tar', f'{request_id}.ready')
-
-    def signed_archive_info_for_request_id(
-            self, request_id: str) -> ArchiveWithIndicator:
-        return self.archive_info_for_request_id(
-            self.signed_storage_dir, request_id)
-
-    def unsigned_archive_info_for_request_id(
-            self, request_id: str) -> ArchiveWithIndicator:
-        return self.archive_info_for_request_id(
-            self.unsigned_storage_dir, request_id)
-
-    ############################################################################
-    # Buildbot worker side helpers.
-
-    @abc.abstractmethod
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        """
-        Check whether file is to be signed.
-
-        Is used by both single file signing pipeline and recursive directory
-        signing pipeline.
-
-        This is where code signer is to check whether file is to be signed or
-        not. This check might be based on a simple extension test or on actual
-        test whether file have a digital signature already or not.
-        """
-
-    def collect_files_to_sign(self, path: Path) \
-            -> List[AbsoluteAndRelativeFileName]:
-        """
-        Get all files which need to be signed from the given path.
-
-        NOTE: The path might either be a file or directory.
-
-        This function is run from the buildbot worker side.
-        """
-
-        # If there is a single file provided trust the buildbot worker that it
-        # is eligible for signing.
-        if path.is_file():
-            file = AbsoluteAndRelativeFileName.from_path(path)
-            if not self.check_file_is_to_be_signed(file):
-                return []
-            return [file]
-
-        all_files = AbsoluteAndRelativeFileName.recursively_from_directory(
-            path)
-        files_to_be_signed = [file for file in all_files
-                              if self.check_file_is_to_be_signed(file)]
-        return files_to_be_signed
-
-    def wait_for_signed_archive_or_die(self, request_id) -> None:
-        """
-        Wait until archive with signed files is available.
-
-        Will only return if the archive with signed files is available. If there
-        was an error during code sign procedure the SystemExit exception is
-        raised, with the message set to the error reported by the codesign
-        server.
-
-        Will only wait for the configured time. If that time exceeds and there
-        is still no responce from the signing server the application will exit
-        with a non-zero exit code.
-
-        """
-
-        signed_archive_info = self.signed_archive_info_for_request_id(
-            request_id)
-        unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-            request_id)
-
-        timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS
-        time_start = time.monotonic()
-        while not signed_archive_info.is_ready():
-            time.sleep(1)
-            time_slept_in_seconds = time.monotonic() - time_start
-            if time_slept_in_seconds > timeout_in_seconds:
-                signed_archive_info.clean()
-                unsigned_archive_info.clean()
-                raise SystemExit("Signing server didn't finish signing in "
-                                 f'{timeout_in_seconds} seconds, dying :(')
-
-        archive_state = signed_archive_info.get_state()
-        if archive_state.has_error():
-            signed_archive_info.clean()
-            unsigned_archive_info.clean()
-            raise SystemExit(
-                f'Error happenned during codesign procedure: {archive_state.error_message}')
-
-    def copy_signed_files_to_directory(
-            self, signed_dir: Path, destination_dir: Path) -> None:
-        """
-        Copy all files from signed_dir to destination_dir.
-
-        This function will overwrite any existing file. Permissions are copied
-        from the source files, but other metadata, such as timestamps, are not.
-        """
-        for signed_filepath in signed_dir.glob('**/*'):
-            if not signed_filepath.is_file():
-                continue
-
-            relative_filepath = signed_filepath.relative_to(signed_dir)
-            destination_filepath = destination_dir / relative_filepath
-            destination_filepath.parent.mkdir(parents=True, exist_ok=True)
-
-            shutil.copy(signed_filepath, destination_filepath)
-
-    def run_buildbot_path_sign_pipeline(self, path: Path) -> None:
-        """
-        Run all steps needed to make given path signed.
-
-        Path points to an unsigned file or a directory which contains unsigned
-        files.
-
-        If the path points to a single file then this file will be signed.
-        This is used to sign a final bundle such as .msi on Windows or .dmg on
-        macOS.
-
-        NOTE: The code signed implementation might actually reject signing the
-        file, in which case the file will be left unsigned. This isn't anything
-        to be considered a failure situation, just might happen when buildbot
-        worker can not detect whether signing is really required in a specific
-        case or not.
-
-        If the path points to a directory then code signer will sign all
-        signable files from it (finding them recursively).
-        """
-
-        self.cleanup_environment_for_builder()
-
-        # Make sure storage directory exists.
-        self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True)
-
-        # Collect all files which needs to be signed and pack them into a single
-        # archive which will be sent to the signing server.
-        logger_builder.info('Collecting files which are to be signed...')
-        files = self.collect_files_to_sign(path)
-        if not files:
-            logger_builder.info('No files to be signed, ignoring.')
-            return
-        logger_builder.info('Found %d files to sign.', len(files))
-
-        request_id = self.generate_request_id()
-        signed_archive_info = self.signed_archive_info_for_request_id(
-            request_id)
-        unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-            request_id)
-
-        pack_files(files=files,
-                   archive_filepath=unsigned_archive_info.archive_filepath)
-        unsigned_archive_info.tag_ready()
-
-        # Wait for the signing server to finish signing.
-        logger_builder.info('Waiting signing server to sign the files...')
-        self.wait_for_signed_archive_or_die(request_id)
-
-        # Extract signed files from archive and move files to final location.
-        with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str:
-            unpacked_signed_files_dir = Path(temp_dir_str)
-
-            logger_builder.info('Extracting signed files from archive...')
-            extract_files(
-                archive_filepath=signed_archive_info.archive_filepath,
-                extraction_dir=unpacked_signed_files_dir)
-
-            destination_dir = path
-            if destination_dir.is_file():
-                destination_dir = destination_dir.parent
-            self.copy_signed_files_to_directory(
-                unpacked_signed_files_dir, destination_dir)
-
-        logger_builder.info('Removing archive with signed files...')
-        signed_archive_info.clean()
-
-    ############################################################################
-    # Signing server side helpers.
-
-    def wait_for_sign_request(self) -> str:
-        """
-        Wait for the buildbot to request signing of an archive.
-
-        Returns an identifier of signing request.
-        """
-
-        # TOOD(sergey): Support graceful shutdown on Ctrl-C.
-
-        logger_server.info(
-            f'Waiting for a request directory {self.unsigned_storage_dir} to appear.')
-        while not self.unsigned_storage_dir.exists():
-            time.sleep(1)
-
-        logger_server.info(
-            'Waiting for a READY indicator of any signing request.')
-        request_id = None
-        while request_id is None:
-            for file in self.unsigned_storage_dir.iterdir():
-                if file.suffix != '.ready':
-                    continue
-                request_id = file.stem
-                logger_server.info(f'Found READY for request ID {request_id}.')
-            if request_id is None:
-                time.sleep(1)
-
-        unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-            request_id)
-        while not unsigned_archive_info.is_ready():
-            time.sleep(1)
-
-        return request_id
-
-    @abc.abstractmethod
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        """
-        Sign all files in the given directory.
-
-        NOTE: Signing should happen in-place.
-        """
-
-    def run_signing_pipeline(self, request_id: str):
-        """
-        Run the full signing pipeline starting from the point when buildbot
-        worker have requested signing.
-        """
-
-        # Make sure storage directory exists.
-        self.signed_storage_dir.mkdir(parents=True, exist_ok=True)
-
-        with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str:
-            temp_dir = Path(temp_dir_str)
-
-            signed_archive_info = self.signed_archive_info_for_request_id(
-                request_id)
-            unsigned_archive_info = self.unsigned_archive_info_for_request_id(
-                request_id)
-
-            logger_server.info('Extracting unsigned files from archive...')
-            extract_files(
-                archive_filepath=unsigned_archive_info.archive_filepath,
-                extraction_dir=temp_dir)
-
-            logger_server.info('Collecting all files which needs signing...')
-            files = AbsoluteAndRelativeFileName.recursively_from_directory(
-                temp_dir)
-
-            logger_server.info('Signing all requested files...')
-            try:
-                self.sign_all_files(files)
-            except CodeSignException as error:
-                signed_archive_info.tag_ready(error_message=error.message)
-                unsigned_archive_info.clean()
-                logger_server.info('Signing is complete with errors.')
-                return
-
-            logger_server.info('Packing signed files...')
-            pack_files(files=files,
-                       archive_filepath=signed_archive_info.archive_filepath)
-            signed_archive_info.tag_ready()
-
-            logger_server.info('Removing signing request...')
-            unsigned_archive_info.clean()
-
-            logger_server.info('Signing is complete.')
-
-    def run_signing_server(self):
-        logger_server.info('Starting new code signing server...')
-        self.cleanup_environment_for_signing_server()
-        logger_server.info('Code signing server is ready')
-        while True:
-            logger_server.info('Waiting for the signing request in %s...',
-                               self.unsigned_storage_dir)
-            request_id = self.wait_for_sign_request()
-
-            logger_server.info(
-                f'Beging signign procedure for request ID {request_id}.')
-            self.run_signing_pipeline(request_id)
-
-    ############################################################################
-    # Command executing.
-    #
-    # Abstracted to a degree that allows to run commands from a foreign
-    # platform.
-    # The goal with this is to allow performing dry-run tests of code signer
-    # server from other platforms (for example, to test that macOS code signer
-    # does what it is supposed to after doing a refactor on Linux).
-
-    # TODO(sergey): What is the type annotation for the command?
-    def run_command_or_mock(self, command, platform: util.Platform) -> None:
-        """
-        Run given command if current platform matches given one
-
-        If the platform is different then it will only be printed allowing
-        to verify logic of the code signing process.
-        """
-
-        if platform != self.platform:
-            logger_server.info(
-                f'Will run command for {platform}: {command}')
-            return
-
-        logger_server.info(f'Running command: {command}')
-        subprocess.run(command)
-
-    # TODO(sergey): What is the type annotation for the command?
-    def check_output_or_mock(self, command,
-                             platform: util.Platform,
-                             allow_nonzero_exit_code=False) -> str:
-        """
-        Run given command if current platform matches given one
-
-        If the platform is different then it will only be printed allowing
-        to verify logic of the code signing process.
-
-        If allow_nonzero_exit_code is truth then the output will be returned
-        even if application quit with non-zero exit code.
-        Otherwise an subprocess.CalledProcessError exception will be raised
-        in such case.
-        """
-
-        if platform != self.platform:
-            logger_server.info(
-                f'Will run command for {platform}: {command}')
-            return
-
-        if allow_nonzero_exit_code:
-            process = subprocess.Popen(command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.STDOUT)
-            output = process.communicate()[0]
-            return output.decode()
-
-        logger_server.info(f'Running command: {command}')
-        return subprocess.check_output(
-            command, stderr=subprocess.STDOUT).decode()

build_files/buildbot/codesign/config_builder.py

@@ -1,62 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Configuration of a code signer which is specific to the code running from
-# buildbot's worker.
-
-import sys
-
-from pathlib import Path
-
-import codesign.util as util
-
-from codesign.config_common import *
-
-platform = util.get_current_platform()
-if platform == util.Platform.LINUX:
-    SHARED_STORAGE_DIR = Path('/data/codesign')
-elif platform == util.Platform.WINDOWS:
-    SHARED_STORAGE_DIR = Path('Z:\\codesign')
-elif platform == util.Platform.MACOS:
-    SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign')
-
-# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
-LOGGING = {
-    'version': 1,
-    'formatters': {
-        'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
-    },
-    'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
-            'formatter': 'default',
-            'stream': 'ext://sys.stderr',
-        }
-    },
-    'loggers': {
-        'codesign': {'level': 'INFO'},
-    },
-    'root': {
-        'level': 'WARNING',
-        'handlers': [
-            'console',
-        ],
-    }
-}

build_files/buildbot/codesign/config_common.py

@@ -1,36 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-from pathlib import Path
-
-# Timeout in seconds for the signing process.
-#
-# This is how long buildbot packing step will wait signing server to
-# perform signing.
-#
-# NOTE: Notarization could take a long time, hence the rather high value
-# here. Might consider using different timeout for different platforms.
-TIMEOUT_IN_SECONDS = 45 * 60 * 60
-
-# Directory which is shared across buildbot worker and signing server.
-#
-# This is where worker puts files requested for signing as well as where
-# server puts signed files.
-SHARED_STORAGE_DIR: Path

build_files/buildbot/codesign/config_server_template.py

@@ -1,101 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Configuration of a code signer which is specific to the code signing server.
-#
-# NOTE: DO NOT put any sensitive information here, put it in an actual
-# configuration on the signing machine.
-
-from pathlib import Path
-
-from codesign.config_common import *
-
-CODESIGN_DIRECTORY = Path(__file__).absolute().parent
-BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent
-
-################################################################################
-# Common configuration.
-
-# Directory where folders for codesign requests and signed result are stored.
-# For example, /data/codesign
-SHARED_STORAGE_DIR: Path
-
-################################################################################
-# macOS-specific configuration.
-
-MACOS_ENTITLEMENTS_FILE = \
-    BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist'
-
-# Identity of the Developer ID Application certificate which is to be used for
-# codesign tool.
-# Use `security find-identity -v -p codesigning` to find the identity.
-#
-# NOTE: This identity is just an example from release/darwin/README.txt.
-MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97'
-
-# User name (Apple ID) which will be used to request notarization.
-MACOS_XCRUN_USERNAME = 'me@example.com'
-
-# One-time application password which will be used to request notarization.
-MACOS_XCRUN_PASSWORD = '@keychain:altool-password'
-
-# Timeout in seconds within which the notarial office is supposed to reply.
-MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60
-
-################################################################################
-# Windows-specific configuration.
-
-# URL to the timestamping authority.
-WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com'
-
-# Full path to the certificate used for signing.
-#
-# The path and expected file format might vary depending on a platform.
-#
-# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look
-# like Path('C:\\Secret\\Blender.pfx').
-WIN_CERTIFICATE_FILEPATH: Path
-
-################################################################################
-# Logging configuration, common for all platforms.
-
-# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
-LOGGING = {
-    'version': 1,
-    'formatters': {
-        'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
-    },
-    'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
-            'formatter': 'default',
-            'stream': 'ext://sys.stderr',
-        }
-    },
-    'loggers': {
-        'codesign': {'level': 'INFO'},
-    },
-    'root': {
-        'level': 'WARNING',
-        'handlers': [
-            'console',
-        ],
-    }
-}

build_files/buildbot/codesign/exception.py

@@ -1,26 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-class CodeSignException(Exception):
-    message: str
-
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)

build_files/buildbot/codesign/linux_code_signer.py

@@ -1,72 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# NOTE: This is a no-op signer (since there isn't really a procedure to sign
-# Linux binaries yet). Used to debug and verify the code signing routines on
-# a Linux environment.
-
-import logging
-
-from pathlib import Path
-from typing import List
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.base_code_signer import BaseCodeSigner
-
-logger = logging.getLogger(__name__)
-logger_server = logger.getChild('server')
-
-
-class LinuxCodeSigner(BaseCodeSigner):
-    def is_active(self) -> bool:
-        """
-        Check whether this signer is active.
-
-        if it is inactive, no files will be signed.
-
-        Is used to be able to debug code signing pipeline on Linux, where there
-        is no code signing happening in the actual buildbot and release
-        environment.
-        """
-        return False
-
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        if file.relative_filepath == Path('blender'):
-            return True
-        if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and
-                file.relative_filepath.name.startwith('python')):
-            return True
-        if file.relative_filepath.suffix == '.so':
-            return True
-        return False
-
-    def collect_files_to_sign(self, path: Path) \
-            -> List[AbsoluteAndRelativeFileName]:
-        if not self.is_active():
-            return []
-
-        return super().collect_files_to_sign(path)
-
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        num_files = len(files)
-        for file_index, file in enumerate(files):
-            logger.info('Server: Signed file [%d/%d] %s',
-                        file_index + 1, num_files, file.relative_filepath)

build_files/buildbot/codesign/macos_code_signer.py

@@ -1,456 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import logging
-import re
-import stat
-import subprocess
-import time
-
-from pathlib import Path
-from typing import List
-
-import codesign.util as util
-
-from buildbot_utils import Builder
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.base_code_signer import BaseCodeSigner
-from codesign.exception import CodeSignException
-
-logger = logging.getLogger(__name__)
-logger_server = logger.getChild('server')
-
-# NOTE: Check is done as filename.endswith(), so keep the dot
-EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'}
-
-# Prefixes of a file (not directory) name which are to be signed.
-# Used to sign extra executable files in Contents/Resources.
-NAME_PREFIXES_TO_BE_SIGNED = {'python'}
-
-
-class NotarizationException(CodeSignException):
-    pass
-
-
-def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool:
-    """
-    Check whether file is coming from an .app bundle
-    """
-    parts = file.relative_filepath.parts
-    if not parts:
-        return False
-    if not parts[0].endswith('.app'):
-        return False
-    return True
-
-
-def get_bundle_from_file(
-        file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName:
-    """
-    Get AbsoluteAndRelativeFileName descriptor of bundle
-    """
-    assert(is_file_from_bundle(file))
-
-    parts = file.relative_filepath.parts
-    bundle_name = parts[0]
-
-    base_dir = file.base_dir
-    bundle_filepath = file.base_dir / bundle_name
-    return AbsoluteAndRelativeFileName(base_dir, bundle_filepath)
-
-
-def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool:
-    """
-    Check whether given file is an executable within an app bundle
-    """
-    if not is_file_from_bundle(file):
-        return False
-
-    parts = file.relative_filepath.parts
-    num_parts = len(parts)
-    if num_parts < 3:
-        return False
-
-    if parts[1:3] != ('Contents', 'MacOS'):
-        return False
-
-    return True
-
-
-def xcrun_field_value_from_output(field: str, output: str) -> str:
-    """
-    Get value of a given field from xcrun output.
-
-    If field is not found empty string is returned.
-    """
-
-    field_prefix = field + ': '
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith(field_prefix):
-            return line[len(field_prefix):]
-    return ''
-
-
-class MacOSCodeSigner(BaseCodeSigner):
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        if file.relative_filepath.name.startswith('.'):
-            return False
-
-        if is_bundle_executable_file(file):
-            return True
-
-        base_name = file.relative_filepath.name
-        if any(base_name.startswith(prefix)
-               for prefix in NAME_PREFIXES_TO_BE_SIGNED):
-            return True
-
-        mode = file.absolute_filepath.lstat().st_mode
-        if mode & stat.S_IXUSR != 0:
-            file_output = subprocess.check_output(
-                ("file", file.absolute_filepath)).decode()
-            if "64-bit executable" in file_output:
-                return True
-
-        return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED
-
-    def collect_files_to_sign(self, path: Path) \
-            -> List[AbsoluteAndRelativeFileName]:
-        # Include all files when signing app or dmg bundle: all the files are
-        # needed to do valid signature of bundle.
-        if path.name.endswith('.app'):
-            return AbsoluteAndRelativeFileName.recursively_from_directory(path)
-        if path.is_dir():
-            files = []
-            for child in path.iterdir():
-                if child.name.endswith('.app'):
-                    current_files = AbsoluteAndRelativeFileName.recursively_from_directory(
-                        child)
-                else:
-                    current_files = super().collect_files_to_sign(child)
-                for current_file in current_files:
-                    files.append(AbsoluteAndRelativeFileName(
-                        path, current_file.absolute_filepath))
-            return files
-        return super().collect_files_to_sign(path)
-
-    ############################################################################
-    # Codesign.
-
-    def codesign_remove_signature(
-            self, file: AbsoluteAndRelativeFileName) -> None:
-        """
-        Make sure given file does not have codesign signature
-
-        This is needed because codesigning is not possible for file which has
-        signature already.
-        """
-
-        logger_server.info(
-            'Removing codesign signature from %s...', file.relative_filepath)
-
-        command = ['codesign', '--remove-signature', file.absolute_filepath]
-        self.run_command_or_mock(command, util.Platform.MACOS)
-
-    def codesign_file(
-            self, file: AbsoluteAndRelativeFileName) -> None:
-        """
-        Sign given file
-
-        NOTE: File must not have any signatures.
-        """
-
-        logger_server.info(
-            'Codesigning %s...', file.relative_filepath)
-
-        entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE
-        command = ['codesign',
-                   '--timestamp',
-                   '--options', 'runtime',
-                   f'--entitlements={entitlements_file}',
-                   '--sign', self.config.MACOS_CODESIGN_IDENTITY,
-                   file.absolute_filepath]
-        self.run_command_or_mock(command, util.Platform.MACOS)
-
-    def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        """
-        Run codesign tool on all eligible files in the given list.
-
-        Will ignore all files which are not to be signed. For the rest will
-        remove possible existing signature and add a new signature.
-        """
-
-        num_files = len(files)
-        have_ignored_files = False
-        signed_files = []
-        for file_index, file in enumerate(files):
-            # Ignore file if it is not to be signed.
-            # Allows to manually construct ZIP of a bundle and get it signed.
-            if not self.check_file_is_to_be_signed(file):
-                logger_server.info(
-                    'Ignoring file [%d/%d] %s',
-                    file_index + 1, num_files, file.relative_filepath)
-                have_ignored_files = True
-                continue
-
-            logger_server.info(
-                'Running codesigning routines for file [%d/%d] %s...',
-                file_index + 1, num_files, file.relative_filepath)
-
-            self.codesign_remove_signature(file)
-            self.codesign_file(file)
-
-            signed_files.append(file)
-
-        if have_ignored_files:
-            logger_server.info('Signed %d files:', len(signed_files))
-            num_signed_files = len(signed_files)
-            for file_index, signed_file in enumerate(signed_files):
-                logger_server.info(
-                    '- [%d/%d] %s',
-                    file_index + 1, num_signed_files,
-                    signed_file.relative_filepath)
-
-    def codesign_bundles(
-            self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        """
-        Codesign all .app bundles in the given list of files.
-
-        Bundle is deducted from paths of the files, and every bundle is only
-        signed once.
-        """
-
-        signed_bundles = set()
-        extra_files = []
-
-        for file in files:
-            if not is_file_from_bundle(file):
-                continue
-            bundle = get_bundle_from_file(file)
-            bundle_name = bundle.relative_filepath
-            if bundle_name in signed_bundles:
-                continue
-
-            logger_server.info('Running codesign routines on bundle %s',
-                               bundle_name)
-
-            # It is not possible to remove signature from DMG.
-            if bundle.relative_filepath.name.endswith('.app'):
-                self.codesign_remove_signature(bundle)
-            self.codesign_file(bundle)
-
-            signed_bundles.add(bundle_name)
-
-            # Codesign on a bundle adds an extra folder with information.
-            # It needs to be compied to the source.
-            code_signature_directory = \
-                bundle.absolute_filepath / 'Contents' / '_CodeSignature'
-            code_signature_files = \
-                AbsoluteAndRelativeFileName.recursively_from_directory(
-                    code_signature_directory)
-            for code_signature_file in code_signature_files:
-                bundle_relative_file = AbsoluteAndRelativeFileName(
-                    bundle.base_dir,
-                    code_signature_directory /
-                    code_signature_file.relative_filepath)
-                extra_files.append(bundle_relative_file)
-
-        files.extend(extra_files)
-
-    ############################################################################
-    # Notarization.
-
-    def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str:
-        """
-        Get bundle ID which will be used to notarize DMG
-        """
-        name = file.relative_filepath.name
-        app_name = name.split('-', 2)[0].lower()
-
-        app_name_words = app_name.split()
-        if len(app_name_words) > 1:
-            app_name_id = ''.join(word.capitalize() for word in app_name_words)
-        else:
-            app_name_id = app_name_words[0]
-
-        # TODO(sergey): Consider using "alpha" for buildbot builds.
-        return f'org.blenderfoundation.{app_name_id}.release'
-
-    def notarize_request(self, file) -> str:
-        """
-        Request notarization of the given file.
-
-        Returns UUID of the notarization request. If error occurred None is
-        returned instead of UUID.
-        """
-
-        bundle_id = self.notarize_get_bundle_id(file)
-        logger_server.info('Bundle ID: %s', bundle_id)
-
-        logger_server.info('Submitting file to the notarial office.')
-        command = [
-            'xcrun', 'altool', '--notarize-app', '--verbose',
-            '-f', file.absolute_filepath,
-            '--primary-bundle-id', bundle_id,
-            '--username', self.config.MACOS_XCRUN_USERNAME,
-            '--password', self.config.MACOS_XCRUN_PASSWORD]
-
-        output = self.check_output_or_mock(
-            command, util.Platform.MACOS, allow_nonzero_exit_code=True)
-
-        for line in output.splitlines():
-            line = line.strip()
-            if line.startswith('RequestUUID = '):
-                request_uuid = line[14:]
-                return request_uuid
-
-            # Check whether the package has been already submitted.
-            if 'The software asset has already been uploaded.' in line:
-                request_uuid = re.sub(
-                    '.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line)
-                logger_server.warning(
-                    f'The package has been already submitted under UUID {request_uuid}')
-                return request_uuid
-
-        logger_server.error(output)
-        logger_server.error('xcrun command did not report RequestUUID')
-        return None
-
-    def notarize_review_status(self, xcrun_output: str) -> bool:
-        """
-        Review status returned by xcrun's notarization info
-
-        Returns truth if the notarization process has finished.
-        If there are errors during notarization, a NotarizationException()
-        exception is thrown with status message from the notarial office.
-        """
-
-        # Parse status and message
-        status = xcrun_field_value_from_output('Status', xcrun_output)
-        status_message = xcrun_field_value_from_output(
-            'Status Message', xcrun_output)
-
-        if status == 'success':
-            logger_server.info(
-                'Package successfully notarized: %s', status_message)
-            return True
-
-        if status == 'invalid':
-            logger_server.error(xcrun_output)
-            logger_server.error(
-                'Package notarization has failed: %s', status_message)
-            raise NotarizationException(status_message)
-
-        if status == 'in progress':
-            return False
-
-        logger_server.info(
-            'Unknown notarization status %s (%s)', status, status_message)
-
-        return False
-
-    def notarize_wait_result(self, request_uuid: str) -> None:
-        """
-        Wait for until notarial office have a reply
-        """
-
-        logger_server.info(
-            'Waiting for a result from the notarization office.')
-
-        command = ['xcrun', 'altool',
-                   '--notarization-info', request_uuid,
-                   '--username', self.config.MACOS_XCRUN_USERNAME,
-                   '--password', self.config.MACOS_XCRUN_PASSWORD]
-
-        time_start = time.monotonic()
-        timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS
-
-        while True:
-            xcrun_output = self.check_output_or_mock(
-                command, util.Platform.MACOS, allow_nonzero_exit_code=True)
-
-            if self.notarize_review_status(xcrun_output):
-                break
-
-            logger_server.info('Keep waiting for notarization office.')
-            time.sleep(30)
-
-            time_slept_in_seconds = time.monotonic() - time_start
-            if time_slept_in_seconds > timeout_in_seconds:
-                logger_server.error(
-                    "Notarial office didn't reply in %f seconds.",
-                    timeout_in_seconds)
-
-    def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool:
-        """
-        Staple notarial label on the file
-        """
-
-        logger_server.info('Stapling notarial stamp.')
-
-        command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath]
-        self.check_output_or_mock(command, util.Platform.MACOS)
-
-    def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool:
-        """
-        Run entire pipeline to get DMG notarized.
-        """
-        logger_server.info('Begin notarization routines on %s',
-                           file.relative_filepath)
-
-        # Submit file for notarization.
-        request_uuid = self.notarize_request(file)
-        if not request_uuid:
-            return False
-        logger_server.info('Received Request UUID: %s', request_uuid)
-
-        # Wait for the status from the notarization office.
-        if not self.notarize_wait_result(request_uuid):
-            return False
-
-        # Staple.
-        self.notarize_staple(file)
-
-    def notarize_all_dmg(
-            self, files: List[AbsoluteAndRelativeFileName]) -> bool:
-        """
-        Notarize all DMG images from the input.
-
-        Images are supposed to be codesigned already.
-        """
-        for file in files:
-            if not file.relative_filepath.name.endswith('.dmg'):
-                continue
-            if not self.check_file_is_to_be_signed(file):
-                continue
-
-            self.notarize_dmg(file)
-
-    ############################################################################
-    # Entry point.
-
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        # TODO(sergey): Handle errors somehow.
-
-        self.codesign_all_files(files)
-        self.codesign_bundles(files)
-        self.notarize_all_dmg(files)

build_files/buildbot/codesign/simple_code_signer.py

@@ -1,52 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-
-import logging.config
-import sys
-
-from pathlib import Path
-from typing import Optional
-
-import codesign.config_builder
-import codesign.util as util
-from codesign.base_code_signer import BaseCodeSigner
-
-
-class SimpleCodeSigner:
-    code_signer: Optional[BaseCodeSigner]
-
-    def __init__(self):
-        platform = util.get_current_platform()
-        if platform == util.Platform.LINUX:
-            from codesign.linux_code_signer import LinuxCodeSigner
-            self.code_signer = LinuxCodeSigner(codesign.config_builder)
-        elif platform == util.Platform.MACOS:
-            from codesign.macos_code_signer import MacOSCodeSigner
-            self.code_signer = MacOSCodeSigner(codesign.config_builder)
-        elif platform == util.Platform.WINDOWS:
-            from codesign.windows_code_signer import WindowsCodeSigner
-            self.code_signer = WindowsCodeSigner(codesign.config_builder)
-        else:
-            self.code_signer = None
-
-    def sign_file_or_directory(self, path: Path) -> None:
-        logging.config.dictConfig(codesign.config_builder.LOGGING)
-        self.code_signer.run_buildbot_path_sign_pipeline(path)

build_files/buildbot/codesign/util.py

@@ -1,54 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import sys
-
-from enum import Enum
-from pathlib import Path
-
-
-class Platform(Enum):
-    LINUX = 1
-    MACOS = 2
-    WINDOWS = 3
-
-
-def get_current_platform() -> Platform:
-    if sys.platform == 'linux':
-        return Platform.LINUX
-    elif sys.platform == 'darwin':
-        return Platform.MACOS
-    elif sys.platform == 'win32':
-        return Platform.WINDOWS
-    raise Exception(f'Unknown platform {sys.platform}')
-
-
-def ensure_file_does_not_exist_or_die(filepath: Path) -> None:
-    """
-    If the file exists, unlink it.
-    If the file path exists and is not a file an assert will trigger.
-    If the file path does not exists nothing happens.
-    """
-    if not filepath.exists():
-        return
-    if not filepath.is_file():
-        # TODO(sergey): Provide information about what the filepath actually is.
-        raise SystemExit(f'{filepath} is expected to be a file, but is not')
-    filepath.unlink()

build_files/buildbot/codesign/windows_code_signer.py

@@ -1,117 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import logging
-
-from pathlib import Path
-from typing import List
-
-import codesign.util as util
-
-from buildbot_utils import Builder
-
-from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
-from codesign.base_code_signer import BaseCodeSigner
-from codesign.exception import CodeSignException
-
-logger = logging.getLogger(__name__)
-logger_server = logger.getChild('server')
-
-# NOTE: Check is done as filename.endswith(), so keep the dot
-EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'}
-
-BLACKLIST_FILE_PREFIXES = (
-    'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime')
-
-
-class SigntoolException(CodeSignException):
-    pass
-
-class WindowsCodeSigner(BaseCodeSigner):
-    def check_file_is_to_be_signed(
-            self, file: AbsoluteAndRelativeFileName) -> bool:
-        base_name = file.relative_filepath.name
-        if any(base_name.startswith(prefix)
-               for prefix in BLACKLIST_FILE_PREFIXES):
-            return False
-
-        return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED
-
-
-    def get_sign_command_prefix(self) -> List[str]:
-        return [
-            'signtool', 'sign', '/v',
-            '/f', self.config.WIN_CERTIFICATE_FILEPATH,
-            '/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL]
-
-
-    def run_codesign_tool(self, filepath: Path) -> None:
-        command = self.get_sign_command_prefix() + [filepath]
-
-        try:
-            codesign_output = self.check_output_or_mock(command, util.Platform.WINDOWS)
-        except subprocess.CalledProcessError as e:
-            raise SigntoolException(f'Error running signtool {e}')
-
-        logger_server.info(f'signtool output:\n{codesign_output}')
-
-        got_number_of_success = False
-
-        for line in codesign_output.split('\n'):
-            line_clean = line.strip()
-            line_clean_lower = line_clean.lower()
-
-            if line_clean_lower.startswith('number of warnings') or \
-               line_clean_lower.startswith('number of errors'):
-                 number = int(line_clean_lower.split(':')[1])
-                 if number != 0:
-                     raise SigntoolException('Non-clean success of signtool')
-
-            if line_clean_lower.startswith('number of files successfully signed'):
-                 got_number_of_success = True
-                 number = int(line_clean_lower.split(':')[1])
-                 if number != 1:
-                     raise SigntoolException('Signtool did not consider codesign a success')
-
-        if not got_number_of_success:
-            raise SigntoolException('Signtool did not report number of files signed')
-
-
-    def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
-        # NOTE: Sign files one by one to avoid possible command line length
-        # overflow (which could happen if we ever decide to sign every binary
-        # in the install folder, for example).
-        #
-        # TODO(sergey): Consider doing batched signing of handful of files in
-        # one go (but only if this actually known to be much faster).
-        num_files = len(files)
-        for file_index, file in enumerate(files):
-            # Ignore file if it is not to be signed.
-            # Allows to manually construct ZIP of package and get it signed.
-            if not self.check_file_is_to_be_signed(file):
-                logger_server.info(
-                    'Ignoring file [%d/%d] %s',
-                    file_index + 1, num_files, file.relative_filepath)
-                continue
-
-            logger_server.info(
-                'Running signtool command for file [%d/%d] %s...',
-                file_index + 1, num_files, file.relative_filepath)
-            self.run_codesign_tool(file.absolute_filepath)

build_files/buildbot/codesign_server_linux.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# NOTE: This is a no-op signer (since there isn't really a procedure to sign
-# Linux binaries yet). Used to debug and verify the code signing routines on
-# a Linux environment.
-
-import logging.config
-from pathlib import Path
-from typing import List
-
-from codesign.linux_code_signer import LinuxCodeSigner
-import codesign.config_server
-
-if __name__ == "__main__":
-    logging.config.dictConfig(codesign.config_server.LOGGING)
-    code_signer = LinuxCodeSigner(codesign.config_server)
-    code_signer.run_signing_server()

build_files/buildbot/codesign_server_macos.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import logging.config
-from pathlib import Path
-from typing import List
-
-from codesign.macos_code_signer import MacOSCodeSigner
-import codesign.config_server
-
-if __name__ == "__main__":
-    entitlements_file = codesign.config_server.MACOS_ENTITLEMENTS_FILE
-    if not entitlements_file.exists():
-        raise SystemExit(
-            'Entitlements file {entitlements_file} does not exist.')
-    if not entitlements_file.is_file():
-        raise SystemExit(
-            'Entitlements file {entitlements_file} is not a file.')
-
-    logging.config.dictConfig(codesign.config_server.LOGGING)
-    code_signer = MacOSCodeSigner(codesign.config_server)
-    code_signer.run_signing_server()

build_files/buildbot/codesign_server_windows.bat

@@ -1,11 +0,0 @@
-@echo off
-
-rem This is an entry point of the codesign server for Windows.
-rem It makes sure that signtool.exe is within the current PATH and can be
-rem used by the Python script.
-
-SETLOCAL
-
-set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH%
-
-codesign_server_windows.py

build_files/buildbot/codesign_server_windows.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Implementation of codesign server for Windows.
-#
-# NOTE: If signtool.exe is not in the PATH use codesign_server_windows.bat
-
-import logging.config
-import shutil
-
-from pathlib import Path
-from typing import List
-
-import codesign.util as util
-
-from codesign.windows_code_signer import WindowsCodeSigner
-import codesign.config_server
-
-if __name__ == "__main__":
-    logging.config.dictConfig(codesign.config_server.LOGGING)
-
-    logger = logging.getLogger(__name__)
-    logger_server = logger.getChild('server')
-
-    # TODO(sergey): Consider moving such sanity checks into
-    # CodeSigner.check_environment_or_die().
-    if not shutil.which('signtool.exe'):
-        if util.get_current_platform() == util.Platform.WINDOWS:
-            raise SystemExit("signtool.exe is not found in %PATH%")
-        logger_server.info(
-            'signtool.exe not found, '
-            'but will not be used on this foreign platform')
-
-    code_signer = WindowsCodeSigner(codesign.config_server)
-    code_signer.run_signing_server()

build_files/buildbot/worker_bundle_dmg.py

@@ -1,551 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-import argparse
-import re
-import shutil
-import subprocess
-import sys
-import time
-
-from pathlib import Path
-from tempfile import TemporaryDirectory, NamedTemporaryFile
-from typing import List
-
-BUILDBOT_DIRECTORY = Path(__file__).absolute().parent
-CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py'
-BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent
-DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin'
-
-
-# Extra size which is added on top of actual files size when estimating size
-# of destination DNG.
-EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024
-
-################################################################################
-# Common utilities
-
-
-def get_directory_size(root_directory: Path) -> int:
-    """
-    Get size of directory on disk
-    """
-
-    total_size = 0
-    for file in root_directory.glob('**/*'):
-        total_size += file.lstat().st_size
-    return total_size
-
-
-################################################################################
-# DMG bundling specific logic
-
-def create_argument_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'source_dir',
-        type=Path,
-        help='Source directory which points to either existing .app bundle'
-             'or to a directory with .app bundles.')
-    parser.add_argument(
-        '--background-image',
-        type=Path,
-        help="Optional background picture which will be set on the DMG."
-             "If not provided default Blender's one is used.")
-    parser.add_argument(
-        '--volume-name',
-        type=str,
-        help='Optional name of a volume which will be used for DMG.')
-    parser.add_argument(
-        '--dmg',
-        type=Path,
-        help='Optional argument which points to a final DMG file name.')
-    parser.add_argument(
-        '--applescript',
-        type=Path,
-        help="Optional path to applescript to set up folder looks of DMG."
-             "If not provided default Blender's one is used.")
-    parser.add_argument(
-        '--codesign',
-        action="store_true",
-        help="Code sign and notarize DMG contents.")
-    return parser
-
-
-def collect_app_bundles(source_dir: Path) -> List[Path]:
-    """
-    Collect all app bundles which are to be put into DMG
-
-    If the source directory points to FOO.app it will be the only app bundle
-    packed.
-
-    Otherwise all .app bundles from given directory are placed to a single
-    DMG.
-    """
-
-    if source_dir.name.endswith('.app'):
-        return [source_dir]
-
-    app_bundles = []
-    for filename in source_dir.glob('*'):
-        if not filename.is_dir():
-            continue
-        if not filename.name.endswith('.app'):
-            continue
-
-        app_bundles.append(filename)
-
-    return app_bundles
-
-
-def collect_and_log_app_bundles(source_dir: Path) -> List[Path]:
-    app_bundles = collect_app_bundles(source_dir)
-
-    if not app_bundles:
-        print('No app bundles found for packing')
-        return
-
-    print(f'Found {len(app_bundles)} to pack:')
-    for app_bundle in app_bundles:
-        print(f'- {app_bundle}')
-
-    return app_bundles
-
-
-def estimate_dmg_size(app_bundles: List[Path]) -> int:
-    """
-    Estimate size of DMG to hold requested app bundles
-
-    The size is based on actual size of all files in all bundles plus some
-    space to compensate for different size-on-disk plus some space to hold
-    codesign signatures.
-
-    Is better to be on a high side since the empty space is compressed, but
-    lack of space might cause silent failures later on.
-    """
-
-    app_bundles_size = 0
-    for app_bundle in app_bundles:
-        app_bundles_size += get_directory_size(app_bundle)
-
-    return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES
-
-
-def copy_app_bundles_to_directory(app_bundles: List[Path],
-                                  directory: Path) -> None:
-    """
-    Copy all bundles to a given directory
-
-    This directory is what the DMG will be created from.
-    """
-    for app_bundle in app_bundles:
-        print(f'Copying {app_bundle.name}...')
-        shutil.copytree(app_bundle, directory / app_bundle.name)
-
-
-def get_main_app_bundle(app_bundles: List[Path]) -> Path:
-    """
-    Get application bundle main for the installation
-    """
-    return app_bundles[0]
-
-
-def create_dmg_image(app_bundles: List[Path],
-                     dmg_filepath: Path,
-                     volume_name: str) -> None:
-    """
-    Create DMG disk image and put app bundles in it
-
-    No DMG configuration or codesigning is happening here.
-    """
-
-    if dmg_filepath.exists():
-        print(f'Removing existing writable DMG {dmg_filepath}...')
-        dmg_filepath.unlink()
-
-    print('Preparing directory with app bundles for the DMG...')
-    with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str:
-        # Copy all bundles to a clean directory.
-        content_dir = Path(content_dir_str)
-        copy_app_bundles_to_directory(app_bundles, content_dir)
-
-        # Estimate size of the DMG.
-        dmg_size = estimate_dmg_size(app_bundles)
-        print(f'Estimated DMG size: {dmg_size:,} bytes.')
-
-        # Create the DMG.
-        print(f'Creating writable DMG {dmg_filepath}')
-        command = ('hdiutil',
-                   'create',
-                   '-size', str(dmg_size),
-                   '-fs', 'HFS+',
-                   '-srcfolder', content_dir,
-                   '-volname', volume_name,
-                   '-format', 'UDRW',
-                   dmg_filepath)
-        subprocess.run(command)
-
-
-def get_writable_dmg_filepath(dmg_filepath: Path):
-    """
-    Get file path for writable DMG image
-    """
-    parent = dmg_filepath.parent
-    return parent / (dmg_filepath.stem + '-temp.dmg')
-
-
-def mount_readwrite_dmg(dmg_filepath: Path) -> None:
-    """
-    Mount writable DMG
-
-    Mounting point would be /Volumes/<volume name>
-    """
-
-    print(f'Mounting read-write DMG ${dmg_filepath}')
-    command = ('hdiutil',
-               'attach', '-readwrite',
-               '-noverify',
-               '-noautoopen',
-               dmg_filepath)
-    subprocess.run(command)
-
-
-def get_mount_directory_for_volume_name(volume_name: str) -> Path:
-    """
-    Get directory under which the volume will be mounted
-    """
-
-    return Path('/Volumes') / volume_name
-
-
-def eject_volume(volume_name: str) -> None:
-    """
-    Eject given volume, if mounted
-    """
-    mount_directory = get_mount_directory_for_volume_name(volume_name)
-    if not mount_directory.exists():
-        return
-    mount_directory_str = str(mount_directory)
-
-    print(f'Ejecting volume {volume_name}')
-
-    # Figure out which device to eject.
-    mount_output = subprocess.check_output(['mount']).decode()
-    device = ''
-    for line in mount_output.splitlines():
-        if f'on {mount_directory_str} (' not in line:
-            continue
-        tokens = line.split(' ', 3)
-        if len(tokens) < 3:
-            continue
-        if tokens[1] != 'on':
-            continue
-        if device:
-            raise Exception(
-                f'Multiple devices found for mounting point {mount_directory}')
-        device = tokens[0]
-
-    if not device:
-        raise Exception(
-            f'No device found for mounting point {mount_directory}')
-
-    print(f'{mount_directory} is mounted as device {device}, ejecting...')
-    subprocess.run(['diskutil', 'eject', device])
-
-
-def copy_background_if_needed(background_image_filepath: Path,
-                              mount_directory: Path) -> None:
-    """
-    Copy background to the DMG
-
-    If the background image is not specified it will not be copied.
-    """
-
-    if not background_image_filepath:
-        print('No background image provided.')
-        return
-
-    print(f'Copying background image {background_image_filepath}')
-
-    destination_dir = mount_directory / '.background'
-    destination_dir.mkdir(exist_ok=True)
-
-    destination_filepath = destination_dir / background_image_filepath.name
-    shutil.copy(background_image_filepath, destination_filepath)
-
-
-def create_applications_link(mount_directory: Path) -> None:
-    """
-    Create link to /Applications in the given location
-    """
-
-    print('Creating link to /Applications')
-
-    command = ('ln', '-s', '/Applications', mount_directory / ' ')
-    subprocess.run(command)
-
-
-def run_applescript(applescript: Path,
-                    volume_name: str,
-                    app_bundles: List[Path],
-                    background_image_filepath: Path) -> None:
-    """
-    Run given applescript to adjust look and feel of the DMG
-    """
-
-    main_app_bundle = get_main_app_bundle(app_bundles)
-
-    with NamedTemporaryFile(
-            mode='w', suffix='.applescript') as temp_applescript:
-        print('Adjusting applescript for volume name...')
-        # Adjust script to the specific volume name.
-        with open(applescript, mode='r') as input:
-            for line in input.readlines():
-                stripped_line = line.strip()
-                if stripped_line.startswith('tell disk'):
-                    line = re.sub('tell disk ".*"',
-                                  f'tell disk "{volume_name}"',
-                                  line)
-                elif stripped_line.startswith('set background picture'):
-                    if not background_image_filepath:
-                        continue
-                    else:
-                        background_image_short = \
-                            '.background:' + background_image_filepath.name
-                        line = re.sub('to file ".*"',
-                                      f'to file "{background_image_short}"',
-                                      line)
-                line = line.replace('blender.app', main_app_bundle.name)
-                temp_applescript.write(line)
-
-        temp_applescript.flush()
-
-        print('Running applescript...')
-        command = ('osascript',  temp_applescript.name)
-        subprocess.run(command)
-
-        print('Waiting for applescript...')
-
-        # NOTE: This is copied from bundle.sh. The exact reason for sleep is
-        # still remained a mystery.
-        time.sleep(5)
-
-
-def codesign(subject: Path):
-    """
-    Codesign file or directory
-
-    NOTE: For DMG it will also notarize.
-    """
-
-    command = (CODESIGN_SCRIPT, subject)
-    subprocess.run(command)
-
-
-def codesign_app_bundles_in_dmg(mount_directory: str) -> None:
-    """
-    Code sign all binaries and bundles in the mounted directory
-    """
-
-    print(f'Codesigning all app bundles in {mount_directory}')
-    codesign(mount_directory)
-
-
-def codesign_and_notarize_dmg(dmg_filepath: Path) -> None:
-    """
-    Run codesign and notarization pipeline on the DMG
-    """
-
-    print(f'Codesigning and notarizing DMG {dmg_filepath}')
-    codesign(dmg_filepath)
-
-
-def compress_dmg(writable_dmg_filepath: Path,
-                 final_dmg_filepath: Path) -> None:
-    """
-    Compress temporary read-write DMG
-    """
-    command = ('hdiutil', 'convert',
-               writable_dmg_filepath,
-               '-format', 'UDZO',
-               '-o', final_dmg_filepath)
-
-    if final_dmg_filepath.exists():
-        print(f'Removing old compressed DMG {final_dmg_filepath}')
-        final_dmg_filepath.unlink()
-
-    print('Compressing disk image...')
-    subprocess.run(command)
-
-
-def create_final_dmg(app_bundles: List[Path],
-                     dmg_filepath: Path,
-                     background_image_filepath: Path,
-                     volume_name: str,
-                     applescript: Path,
-                     codesign: bool) -> None:
-    """
-    Create DMG with all app bundles
-
-    Will take care configuring background, signing all binaries and app bundles
-    and notarizing the DMG.
-    """
-
-    print('Running all routines to create final DMG')
-
-    writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath)
-    mount_directory = get_mount_directory_for_volume_name(volume_name)
-
-    # Make sure volume is not mounted.
-    # If it is mounted it will prevent removing old DMG files and could make
-    # it so app bundles are copied to the wrong place.
-    eject_volume(volume_name)
-
-    create_dmg_image(app_bundles, writable_dmg_filepath, volume_name)
-
-    mount_readwrite_dmg(writable_dmg_filepath)
-
-    # Run codesign first, prior to copying amything else.
-    #
-    # This allows to recurs into the content of bundles without worrying about
-    # possible interfereice of Application symlink.
-    if codesign:
-        codesign_app_bundles_in_dmg(mount_directory)
-
-    copy_background_if_needed(background_image_filepath, mount_directory)
-    create_applications_link(mount_directory)
-    run_applescript(applescript, volume_name, app_bundles,
-                    background_image_filepath)
-
-    print('Ejecting read-write DMG image...')
-    eject_volume(volume_name)
-
-    compress_dmg(writable_dmg_filepath, dmg_filepath)
-    writable_dmg_filepath.unlink()
-
-    if codesign:
-        codesign_and_notarize_dmg(dmg_filepath)
-
-
-def ensure_dmg_extension(filepath: Path) -> Path:
-    """
-    Make sure given file have .dmg extension
-    """
-
-    if filepath.suffix != '.dmg':
-        return filepath.with_suffix(f'{filepath.suffix}.dmg')
-    return filepath
-
-
-def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path:
-    """
-    Get full file path for the final DMG image
-
-    Will use the provided one when possible, otherwise will deduct it from
-    app bundles.
-
-    If the name is deducted, the DMG is stored in the current directory.
-    """
-
-    if requested_name:
-        return ensure_dmg_extension(requested_name.absolute())
-
-    # TODO(sergey): This is not necessarily the main one.
-    main_bundle = app_bundles[0]
-    # Strip .app from the name
-    return Path(main_bundle.name[:-4] + '.dmg').absolute()
-
-
-def get_background_image(requested_background_image: Path) -> Path:
-    """
-    Get effective filepath for the background image
-    """
-
-    if requested_background_image:
-        return requested_background_image.absolute()
-
-    return DARWIN_DIRECTORY / 'background.tif'
-
-
-def get_applescript(requested_applescript: Path) -> Path:
-    """
-    Get effective filepath for the applescript
-    """
-
-    if requested_applescript:
-        return requested_applescript.absolute()
-
-    return DARWIN_DIRECTORY / 'blender.applescript'
-
-
-def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str:
-    """
-    Deduct volume name from the DMG path
-
-    Will use first part of the DMG file name prior to dash.
-    """
-
-    tokens = dmg_filepath.stem.split('-')
-    words = tokens[0].split()
-
-    return ' '.join(word.capitalize() for word in words)
-
-
-def get_volume_name(requested_volume_name: str,
-                    dmg_filepath: Path) -> str:
-    """
-    Get effective name for DMG volume
-    """
-
-    if requested_volume_name:
-        return requested_volume_name
-
-    return get_volume_name_from_dmg_filepath(dmg_filepath)
-
-
-def main():
-    parser = create_argument_parser()
-    args = parser.parse_args()
-
-    # Get normalized input parameters.
-    source_dir = args.source_dir.absolute()
-    background_image_filepath = get_background_image(args.background_image)
-    applescript = get_applescript(args.applescript)
-    codesign = args.codesign
-
-    app_bundles = collect_and_log_app_bundles(source_dir)
-    if not app_bundles:
-        return
-
-    dmg_filepath = get_dmg_filepath(args.dmg, app_bundles)
-    volume_name = get_volume_name(args.volume_name, dmg_filepath)
-
-    print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)')
-
-    create_final_dmg(app_bundles,
-                     dmg_filepath,
-                     background_image_filepath,
-                     volume_name,
-                     applescript,
-                     codesign)
-
-
-if __name__ == "__main__":
-    main()

build_files/buildbot/worker_codesign.cmake

@@ -1,44 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# This is a script which is used as POST-INSTALL one for regular CMake's
-# INSTALL target.
-# It is used by buildbot workers to sign every binary which is going into
-# the final buundle.
-
-# On Windows Python 3 there only is python.exe, no python3.exe.
-#
-# On other platforms it is possible to have python2 and python3, and a
-# symbolic link to python to either of them. So on those platforms use
-# an explicit Python version.
-if(WIN32)
-  set(PYTHON_EXECUTABLE python)
-else()
-  set(PYTHON_EXECUTABLE python3)
-endif()
-
-execute_process(
-  COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/worker_codesign.py"
-          "${CMAKE_INSTALL_PREFIX}"
-  WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-  RESULT_VARIABLE exit_code
-)
-
-if(NOT exit_code EQUAL "0")
-    message(FATAL_ERROR "Non-zero exit code of codesign tool")
-endif()

build_files/buildbot/worker_codesign.py

@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# Helper script which takes care of signing provided location.
-#
-# The location can either be a directory (in which case all eligible binaries
-# will be signed) or a single file (in which case a single file will be signed).
-#
-# This script takes care of all the complexity of communicating between process
-# which requests file to be signed and the code signing server.
-#
-# NOTE: Signing happens in-place.
-
-import argparse
-import sys
-
-from pathlib import Path
-
-from codesign.simple_code_signer import SimpleCodeSigner
-
-
-def create_argument_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('path_to_sign', type=Path)
-    return parser
-
-
-def main():
-    parser = create_argument_parser()
-    args = parser.parse_args()
-    path_to_sign = args.path_to_sign.absolute()
-
-    if sys.platform == 'win32':
-        # When WIX packed is used to generate .msi on Windows the CPack will
-        # install two different projects and install them to different
-        # installation prefix:
-        #
-        # - C:\b\build\_CPack_Packages\WIX\Blender
-        # - C:\b\build\_CPack_Packages\WIX\Unspecified
-        #
-        # Annoying part is: CMake's post-install script will only be run
-        # once, with the install prefix which corresponds to a project which
-        # was installed last. But we want to sign binaries from all projects.
-        # So in order to do so we detect that we are running for a CPack's
-        # project used for WIX and force parent directory (which includes both
-        # projects) to be signed.
-        #
-        # Here we force both projects to be signed.
-        if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign):
-            path_to_sign = path_to_sign.parent
-
-    code_signer = SimpleCodeSigner()
-    code_signer.sign_file_or_directory(path_to_sign)
-
-
-if __name__ == "__main__":
-    main()

build_files/buildbot/worker_compile.py

@@ -1,135 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import os
-import shutil
-
-import buildbot_utils
-
-
-def get_cmake_options(builder):
-    codesign_script = os.path.join(
-        builder.blender_dir, 'build_files', 'buildbot', 'worker_codesign.cmake')
-
-    config_file = "build_files/cmake/config/blender_release.cmake"
-    options = ['-DCMAKE_BUILD_TYPE:STRING=Release',
-               '-DWITH_GTESTS=ON']
-
-    if builder.platform == 'mac':
-        options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
-        options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
-    elif builder.platform == 'win':
-        options.extend(['-G', 'Visual Studio 16 2019', '-A', 'x64'])
-        if builder.codesign:
-            options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + codesign_script])
-    elif builder.platform == 'linux':
-        config_file = "build_files/buildbot/config/blender_linux.cmake"
-
-    optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK-7.1')
-    options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir)
-
-    # Workaround to build sm_30 kernels with CUDA 10, since CUDA 11 no longer supports that architecture
-    if builder.platform == 'win':
-        options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1')
-        options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe')
-        options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1')
-        options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1/bin/nvcc.exe')
-    elif builder.platform == 'linux':
-        options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-10.1')
-        options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-10.1/bin/nvcc')
-        options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-11.1')
-        options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-11.1/bin/nvcc')
-
-    options.append("-C" + os.path.join(builder.blender_dir, config_file))
-    options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir))
-
-    return options
-
-
-def update_git(builder):
-    # Do extra git fetch because not all platform/git/buildbot combinations
-    # update the origin remote, causing buildinfo to detect local changes.
-    os.chdir(builder.blender_dir)
-
-    print("Fetching remotes")
-    command = ['git', 'fetch', '--all']
-    buildbot_utils.call(builder.command_prefix + command)
-
-
-def clean_directories(builder):
-    # Make sure no garbage remained from the previous run
-    if os.path.isdir(builder.install_dir):
-        shutil.rmtree(builder.install_dir)
-
-    # Make sure build directory exists and enter it
-    os.makedirs(builder.build_dir, exist_ok=True)
-
-    # Remove buildinfo files to force buildbot to re-generate them.
-    for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
-        full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo)
-        if os.path.exists(full_path):
-            print("Removing {}" . format(buildinfo))
-            os.remove(full_path)
-
-
-def cmake_configure(builder):
-    # CMake configuration
-    os.chdir(builder.build_dir)
-
-    cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt')
-    if os.path.exists(cmake_cache):
-        print("Removing CMake cache")
-        os.remove(cmake_cache)
-
-    print("CMake configure:")
-    cmake_options = get_cmake_options(builder)
-    command = ['cmake', builder.blender_dir] + cmake_options
-    buildbot_utils.call(builder.command_prefix + command)
-
-
-def cmake_build(builder):
-    # CMake build
-    os.chdir(builder.build_dir)
-
-    # NOTE: CPack will build an INSTALL target, which would mean that code
-    # signing will happen twice when using `make install` and CPack.
-    # The tricky bit here is that it is not possible to know whether INSTALL
-    # target is used by CPack or by a buildbot itaself. Extra level on top of
-    # this is that on Windows it is required to build INSTALL target in order
-    # to have unit test binaries to run.
-    # So on the one hand we do an extra unneeded code sign on Windows, but on
-    # a positive side we don't add complexity and don't make build process more
-    # fragile trying to avoid this. The signing process is way faster than just
-    # a clean build of buildbot, especially with regression tests enabled.
-    if builder.platform == 'win':
-        command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release']
-    else:
-        command = ['make', '-s', '-j16', 'install']
-
-    print("CMake build:")
-    buildbot_utils.call(builder.command_prefix + command)
-
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-    update_git(builder)
-    clean_directories(builder)
-    cmake_configure(builder)
-    cmake_build(builder)

build_files/buildbot/worker_pack.py

@@ -1,208 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-# Runs on buildbot worker, creating a release package using the build
-# system and zipping it into buildbot_upload.zip. This is then uploaded
-# to the master in the next buildbot step.
-
-import os
-import sys
-
-from pathlib import Path
-
-import buildbot_utils
-
-
-def get_package_name(builder, platform=None):
-    info = buildbot_utils.VersionInfo(builder)
-
-    package_name = 'blender-' + info.full_version
-    if platform:
-        package_name += '-' + platform
-    if not (builder.branch == 'master' or builder.is_release_branch):
-        if info.is_development_build:
-            package_name = builder.branch + "-" + package_name
-
-    return package_name
-
-
-def sign_file_or_directory(path):
-    from codesign.simple_code_signer import SimpleCodeSigner
-    code_signer = SimpleCodeSigner()
-    code_signer.sign_file_or_directory(Path(path))
-
-
-def create_buildbot_upload_zip(builder, package_files):
-    import zipfile
-
-    buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
-    if os.path.exists(buildbot_upload_zip):
-        os.remove(buildbot_upload_zip)
-
-    try:
-        z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
-        for filepath, filename in package_files:
-            print("Packaged", filename)
-            z.write(filepath, arcname=filename)
-        z.close()
-    except Exception as ex:
-        sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
-        sys.exit(1)
-
-
-def create_tar_xz(src, dest, package_name):
-    # One extra to remove leading os.sep when cleaning root for package_root
-    ln = len(src) + 1
-    flist = list()
-
-    # Create list of tuples containing file and archive name
-    for root, dirs, files in os.walk(src):
-        package_root = os.path.join(package_name, root[ln:])
-        flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
-
-    import tarfile
-
-    # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
-    # user compiled the package. If root then unpacks it to /usr/local/ you get
-    # a security issue.
-    def _fakeroot(tarinfo):
-        tarinfo.gid = 0
-        tarinfo.gname = "root"
-        tarinfo.uid = 0
-        tarinfo.uname = "root"
-        return tarinfo
-
-    package = tarfile.open(dest, 'w:xz', preset=9)
-    for entry in flist:
-        package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
-    package.close()
-
-
-def cleanup_files(dirpath, extension):
-    for f in os.listdir(dirpath):
-        filepath = os.path.join(dirpath, f)
-        if os.path.isfile(filepath) and f.endswith(extension):
-            os.remove(filepath)
-
-
-def pack_mac(builder):
-    info = buildbot_utils.VersionInfo(builder)
-
-    os.chdir(builder.build_dir)
-    cleanup_files(builder.build_dir, '.dmg')
-
-    package_name = get_package_name(builder, 'macOS')
-    package_filename = package_name + '.dmg'
-    package_filepath = os.path.join(builder.build_dir, package_filename)
-
-    release_dir = os.path.join(builder.blender_dir, 'release', 'darwin')
-    buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot')
-    bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py')
-
-    command = [bundle_script]
-    command += ['--dmg', package_filepath]
-    if info.is_development_build:
-        background_image = os.path.join(release_dir, 'buildbot', 'background.tif')
-        command += ['--background-image', background_image]
-    if builder.codesign:
-        command += ['--codesign']
-    command += [builder.install_dir]
-    buildbot_utils.call(command)
-
-    create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
-
-
-def pack_win(builder):
-    info = buildbot_utils.VersionInfo(builder)
-
-    os.chdir(builder.build_dir)
-    cleanup_files(builder.build_dir, '.zip')
-
-    # CPack will add the platform name
-    cpack_name = get_package_name(builder, None)
-    package_name = get_package_name(builder, 'windows' + str(builder.bits))
-
-    command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.']
-    buildbot_utils.call(builder.command_prefix + command)
-    command = ['cpack', '-G', 'ZIP']
-    buildbot_utils.call(builder.command_prefix + command)
-
-    package_filename = package_name + '.zip'
-    package_filepath = os.path.join(builder.build_dir, package_filename)
-    package_files = [(package_filepath, package_filename)]
-
-    if info.version_cycle == 'release':
-        # Installer only for final release builds, otherwise will get
-        # 'this product is already installed' messages.
-        command = ['cpack', '-G', 'WIX']
-        buildbot_utils.call(builder.command_prefix + command)
-
-        package_filename = package_name + '.msi'
-        package_filepath = os.path.join(builder.build_dir, package_filename)
-        if builder.codesign:
-            sign_file_or_directory(package_filepath)
-
-        package_files += [(package_filepath, package_filename)]
-
-    create_buildbot_upload_zip(builder, package_files)
-
-
-def pack_linux(builder):
-    blender_executable = os.path.join(builder.install_dir, 'blender')
-
-    info = buildbot_utils.VersionInfo(builder)
-
-    # Strip all unused symbols from the binaries
-    print("Stripping binaries...")
-    buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
-
-    print("Stripping python...")
-    py_target = os.path.join(builder.install_dir, info.short_version)
-    buildbot_utils.call(
-        builder.command_prefix + [
-            'find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';',
-        ],
-    )
-
-    # Construct package name
-    platform_name = 'linux64'
-    package_name = get_package_name(builder, platform_name)
-    package_filename = package_name + ".tar.xz"
-
-    print("Creating .tar.xz archive")
-    package_filepath = builder.install_dir + '.tar.xz'
-    create_tar_xz(builder.install_dir, package_filepath, package_name)
-
-    # Create buildbot_upload.zip
-    create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
-
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-
-    # Make sure install directory always exists
-    os.makedirs(builder.install_dir, exist_ok=True)
-
-    if builder.platform == 'mac':
-        pack_mac(builder)
-    elif builder.platform == 'win':
-        pack_win(builder)
-    elif builder.platform == 'linux':
-        pack_linux(builder)

build_files/buildbot/worker_test.py

@@ -1,42 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import buildbot_utils
-import os
-import sys
-
-
-def get_ctest_arguments(builder):
-    args = ['--output-on-failure']
-    if builder.platform == 'win':
-        args += ['-C', 'Release']
-    return args
-
-
-def test(builder):
-    os.chdir(builder.build_dir)
-
-    command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder)
-    buildbot_utils.call(command)
-
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-    test(builder)

build_files/buildbot/worker_update.py

@@ -1,31 +0,0 @@
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-import buildbot_utils
-import os
-import sys
-
-if __name__ == "__main__":
-    builder = buildbot_utils.create_builder_from_arguments()
-    os.chdir(builder.blender_dir)
-
-    # Run make update which handles all libraries and submodules.
-    make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py")
-    buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"])

build_files/cmake/buildinfo.cmake

@@ -79,7 +79,7 @@ if(EXISTS ${SOURCE_DIR}/.git)
                     ERROR_QUIET)
     if(NOT _git_below_check STREQUAL "")
       # If there're commits between HEAD and upstream this means
-      # that we're reset-ed to older revision. Use it's hash then.
+      # that we're reset-ed to older revision. Use its hash then.
       execute_process(COMMAND git rev-parse --short=12 HEAD
                       WORKING_DIRECTORY ${SOURCE_DIR}
                       OUTPUT_VARIABLE MY_WC_HASH

build_files/cmake/config/blender_release.cmake

@@ -56,10 +56,6 @@ set(WITH_TBB                 ON  CACHE BOOL "" FORCE)
 set(WITH_USD                 ON  CACHE BOOL "" FORCE)
 
 set(WITH_MEM_JEMALLOC          ON  CACHE BOOL "" FORCE)
-set(WITH_CYCLES_CUDA_BINARIES  ON  CACHE BOOL "" FORCE)
-set(WITH_CYCLES_CUBIN_COMPILER OFF CACHE BOOL "" FORCE)
-set(CYCLES_CUDA_BINARIES_ARCH sm_30;sm_35;sm_37;sm_50;sm_52;sm_60;sm_61;sm_70;sm_75;sm_86;compute_75 CACHE STRING "" FORCE)
-set(WITH_CYCLES_DEVICE_OPTIX   ON CACHE BOOL "" FORCE)
 
 # platform dependent options
 if(APPLE)
@@ -80,4 +76,8 @@ if(UNIX AND NOT APPLE)
 endif()
 if(NOT APPLE)
   set(WITH_XR_OPENXR              ON  CACHE BOOL "" FORCE)
+
+  set(WITH_CYCLES_DEVICE_OPTIX    ON  CACHE BOOL "" FORCE)
+  set(WITH_CYCLES_CUDA_BINARIES   ON  CACHE BOOL "" FORCE)
+  set(WITH_CYCLES_CUBIN_COMPILER  OFF CACHE BOOL "" FORCE)
 endif()

build_files/cmake/packaging.cmake

@@ -104,8 +104,8 @@ if(WIN32)
   set(CPACK_WIX_LIGHT_EXTRA_FLAGS -dcl:medium)
 endif()
 
-set(CPACK_PACKAGE_EXECUTABLES "blender" "blender")
-set(CPACK_CREATE_DESKTOP_LINKS "blender" "blender")
+set(CPACK_PACKAGE_EXECUTABLES "blender-launcher" "blender")
+set(CPACK_CREATE_DESKTOP_LINKS "blender-launcher" "blender")
 
 include(CPack)
 

build_files/cmake/platform/platform_apple.cmake

@@ -20,12 +20,6 @@
 
 # Libraries configuration for Apple.
 
-if("${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
-  set(MACOSX_DEPLOYMENT_TARGET 11.00)
-else()
-  set(MACOSX_DEPLOYMENT_TARGET 10.13)
-endif()
-
 macro(find_package_wrapper)
 # do nothing, just satisfy the macro
 endmacro()

build_files/cmake/platform/platform_apple_xcode.cmake

@@ -168,21 +168,15 @@ endif()
 unset(OSX_SDKROOT)
 
 
-# 10.13 is our min. target, if you use higher sdk, weak linking happens
 if("${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
+  # M1 chips run Big Sur onwards.
   set(OSX_MIN_DEPLOYMENT_TARGET 11.00)
 else()
+  # 10.13 is our min. target, if you use higher sdk, weak linking happens
   set(OSX_MIN_DEPLOYMENT_TARGET 10.13)
 endif()
 
-if(CMAKE_OSX_DEPLOYMENT_TARGET)
-  if(${CMAKE_OSX_DEPLOYMENT_TARGET} VERSION_LESS ${OSX_MIN_DEPLOYMENT_TARGET})
-    message(STATUS "Setting deployment target to ${OSX_MIN_DEPLOYMENT_TARGET}, lower versions are not supported")
-    set(CMAKE_OSX_DEPLOYMENT_TARGET "${OSX_MIN_DEPLOYMENT_TARGET}" CACHE STRING "" FORCE)
-  endif()
-else()
-  set(CMAKE_OSX_DEPLOYMENT_TARGET "${OSX_MIN_DEPLOYMENT_TARGET}" CACHE STRING "" FORCE)
-endif()
+set(CMAKE_OSX_DEPLOYMENT_TARGET "${OSX_MIN_DEPLOYMENT_TARGET}" CACHE STRING "" FORCE)
 
 if(NOT ${CMAKE_GENERATOR} MATCHES "Xcode")
   # Force CMAKE_OSX_DEPLOYMENT_TARGET for makefiles, will not work else (CMake bug?)

build_files/cmake/platform/platform_win32.cmake

@@ -119,7 +119,7 @@ string(APPEND CMAKE_MODULE_LINKER_FLAGS " /SAFESEH:NO /ignore:4099")
 list(APPEND PLATFORM_LINKLIBS
   ws2_32 vfw32 winmm kernel32 user32 gdi32 comdlg32 Comctl32 version
   advapi32 shfolder shell32 ole32 oleaut32 uuid psapi Dbghelp Shlwapi
-  pathcch
+  pathcch Shcore
 )
 
 if(WITH_INPUT_IME)
@@ -144,8 +144,8 @@ add_definitions(-D_ALLOW_KEYWORD_MACROS)
 # that both /GR and /GR- are specified.
 remove_cc_flag("/GR")
 
-# We want to support Windows 7 level ABI
-add_definitions(-D_WIN32_WINNT=0x601)
+# Make the Windows 8.1 API available for use.
+add_definitions(-D_WIN32_WINNT=0x603)
 include(build_files/cmake/platform/platform_win32_bundle_crt.cmake)
 remove_cc_flag("/MDd" "/MD" "/Zi")
 

build_files/config/README.md

@@ -0,0 +1,8 @@
+Pipeline Config
+===============
+
+This configuration file is used by buildbot new pipeline for the `update-code` step.
+
+It will soon be used by the ../utils/make_update.py script.
+
+Both buildbot and developers will eventually use the same configuration file.

build_files/config/pipeline_config.json

@@ -0,0 +1,87 @@
+{
+    "update-code":
+    {
+        "git" :
+        {
+            "submodules":
+            [
+                { "path": "release/scripts/addons", "branch": "master", "commit_id": "HEAD" },
+                { "path": "release/scripts/addons_contrib", "branch": "master", "commit_id": "HEAD" },
+                { "path": "release/datafiles/locale", "branch": "master", "commit_id": "HEAD" },
+                { "path": "source/tools", "branch": "master", "commit_id": "HEAD" }
+            ]
+        },
+        "svn": 
+        {
+            "tests": { "path": "lib/tests", "branch": "trunk", "commit_id": "HEAD" },
+            "libraries":
+            {
+                "darwin-x86_64": { "path": "lib/darwin", "branch": "trunk", "commit_id": "HEAD" },
+                "darwin-arm64": { "path": "lib/darwin_arm64", "branch": "trunk", "commit_id": "HEAD" },
+                "linux-x86_64": { "path": "lib/linux_centos7_x86_64", "branch": "trunk", "commit_id": "HEAD" },
+                "windows-amd64": { "path": "lib/win64_vc15", "branch": "trunk", "commit_id": "HEAD" }
+            }
+        }
+    },
+    "buildbot":
+    {
+        "gcc":
+        {
+            "version": "9.0"
+        },
+        "sdks":
+        {
+            "optix":
+            {
+                "version": "7.1.0"
+            },
+            "cuda10":
+            {
+                "version": "10.1"
+            },
+            "cuda11":
+            {
+                "version": "11.3"
+            }
+        },
+        "cmake":
+        {
+            "default":
+            { 
+                "version": "any",
+                "overrides":
+                {
+
+                }
+            },
+            "darwin-x86_64":
+            {
+                "overrides":
+                {
+    
+                }
+            },
+            "darwin-arm64":
+            {
+                "overrides":
+                {
+    
+                }
+            },
+            "linux-x86_64":
+            {
+                "overrides":
+                {
+    
+                }
+            },
+            "windows-amd64":
+            {
+                "overrides":
+                {
+    
+                }
+            }
+        }
+    }
+}

build_files/utils/README.md

@@ -0,0 +1,5 @@
+Make Utility Scripts
+====================
+
+Scripts used only by developers for now
+

doc/blender_file_format/BlendFileReader.py

@@ -85,7 +85,7 @@ def openBlendFile(filename):
     '''
     handle = open(filename, 'rb')
     magic = ReadString(handle, 7)
-    if magic in ("BLENDER", "BULLETf"):
+    if magic in {"BLENDER", "BULLETf"}:
         log.debug("normal blendfile detected")
         handle.seek(0, os.SEEK_SET)
         return handle
@@ -137,7 +137,7 @@ class BlendFile:
         fileblock = BlendFileBlock(handle, self)
         found_dna_block = False
         while not found_dna_block:
-            if fileblock.Header.Code in ("DNA1", "SDNA"):
+            if fileblock.Header.Code in {"DNA1", "SDNA"}:
                 self.Catalog = DNACatalog(self.Header, handle)
                 found_dna_block = True
             else:

doc/python_api/examples/bpy.types.RenderEngine.py

@@ -4,7 +4,9 @@ Simple Render Engine
 """
 
 import bpy
-import bgl
+import array
+import gpu
+from gpu_extras.presets import draw_texture_2d
 
 
 class CustomRenderEngine(bpy.types.RenderEngine):
@@ -100,8 +102,7 @@ class CustomRenderEngine(bpy.types.RenderEngine):
         dimensions = region.width, region.height
 
         # Bind shader that converts from scene linear to display space,
-        bgl.glEnable(bgl.GL_BLEND)
-        bgl.glBlendFunc(bgl.GL_ONE, bgl.GL_ONE_MINUS_SRC_ALPHA)
+        gpu.state.blend_set('ALPHA_PREMULT')
         self.bind_display_space_shader(scene)
 
         if not self.draw_data or self.draw_data.dimensions != dimensions:
@@ -110,7 +111,7 @@ class CustomRenderEngine(bpy.types.RenderEngine):
         self.draw_data.draw()
 
         self.unbind_display_space_shader()
-        bgl.glDisable(bgl.GL_BLEND)
+        gpu.state.blend_set('NONE')
 
 
 class CustomDrawData:
@@ -119,68 +120,21 @@ class CustomDrawData:
         self.dimensions = dimensions
         width, height = dimensions
 
-        pixels = [0.1, 0.2, 0.1, 1.0] * width * height
-        pixels = bgl.Buffer(bgl.GL_FLOAT, width * height * 4, pixels)
+        pixels = width * height * array.array('f', [0.1, 0.2, 0.1, 1.0])
+        pixels = gpu.types.Buffer('FLOAT', width * height * 4, pixels)
 
         # Generate texture
-        self.texture = bgl.Buffer(bgl.GL_INT, 1)
-        bgl.glGenTextures(1, self.texture)
-        bgl.glActiveTexture(bgl.GL_TEXTURE0)
-        bgl.glBindTexture(bgl.GL_TEXTURE_2D, self.texture[0])
-        bgl.glTexImage2D(bgl.GL_TEXTURE_2D, 0, bgl.GL_RGBA16F, width, height, 0, bgl.GL_RGBA, bgl.GL_FLOAT, pixels)
-        bgl.glTexParameteri(bgl.GL_TEXTURE_2D, bgl.GL_TEXTURE_MIN_FILTER, bgl.GL_LINEAR)
-        bgl.glTexParameteri(bgl.GL_TEXTURE_2D, bgl.GL_TEXTURE_MAG_FILTER, bgl.GL_LINEAR)
-        bgl.glBindTexture(bgl.GL_TEXTURE_2D, 0)
+        self.texture = gpu.types.GPUTexture((width, height), format='RGBA16F', data=pixels)
 
-        # Bind shader that converts from scene linear to display space,
-        # use the scene's color management settings.
-        shader_program = bgl.Buffer(bgl.GL_INT, 1)
-        bgl.glGetIntegerv(bgl.GL_CURRENT_PROGRAM, shader_program)
-
-        # Generate vertex array
-        self.vertex_array = bgl.Buffer(bgl.GL_INT, 1)
-        bgl.glGenVertexArrays(1, self.vertex_array)
-        bgl.glBindVertexArray(self.vertex_array[0])
-
-        texturecoord_location = bgl.glGetAttribLocation(shader_program[0], "texCoord")
-        position_location = bgl.glGetAttribLocation(shader_program[0], "pos")
-
-        bgl.glEnableVertexAttribArray(texturecoord_location)
-        bgl.glEnableVertexAttribArray(position_location)
-
-        # Generate geometry buffers for drawing textured quad
-        position = [0.0, 0.0, width, 0.0, width, height, 0.0, height]
-        position = bgl.Buffer(bgl.GL_FLOAT, len(position), position)
-        texcoord = [0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0]
-        texcoord = bgl.Buffer(bgl.GL_FLOAT, len(texcoord), texcoord)
-
-        self.vertex_buffer = bgl.Buffer(bgl.GL_INT, 2)
-
-        bgl.glGenBuffers(2, self.vertex_buffer)
-        bgl.glBindBuffer(bgl.GL_ARRAY_BUFFER, self.vertex_buffer[0])
-        bgl.glBufferData(bgl.GL_ARRAY_BUFFER, 32, position, bgl.GL_STATIC_DRAW)
-        bgl.glVertexAttribPointer(position_location, 2, bgl.GL_FLOAT, bgl.GL_FALSE, 0, None)
-
-        bgl.glBindBuffer(bgl.GL_ARRAY_BUFFER, self.vertex_buffer[1])
-        bgl.glBufferData(bgl.GL_ARRAY_BUFFER, 32, texcoord, bgl.GL_STATIC_DRAW)
-        bgl.glVertexAttribPointer(texturecoord_location, 2, bgl.GL_FLOAT, bgl.GL_FALSE, 0, None)
-
-        bgl.glBindBuffer(bgl.GL_ARRAY_BUFFER, 0)
-        bgl.glBindVertexArray(0)
+        # Note: This is just a didactic example.
+        # In this case it would be more convenient to fill the texture with:
+        # self.texture.clear('FLOAT', value=[0.1, 0.2, 0.1, 1.0])
 
     def __del__(self):
-        bgl.glDeleteBuffers(2, self.vertex_buffer)
-        bgl.glDeleteVertexArrays(1, self.vertex_array)
-        bgl.glBindTexture(bgl.GL_TEXTURE_2D, 0)
-        bgl.glDeleteTextures(1, self.texture)
+        del self.texture
 
     def draw(self):
-        bgl.glActiveTexture(bgl.GL_TEXTURE0)
-        bgl.glBindTexture(bgl.GL_TEXTURE_2D, self.texture[0])
-        bgl.glBindVertexArray(self.vertex_array[0])
-        bgl.glDrawArrays(bgl.GL_TRIANGLE_FAN, 0, 4)
-        bgl.glBindVertexArray(0)
-        bgl.glBindTexture(bgl.GL_TEXTURE_2D, 0)
+        draw_texture_2d(self.texture, (0, 0), self.texture.width, self.texture.height)
 
 
 # RenderEngines also need to tell UI Panels that they are compatible with.

doc/python_api/examples/mathutils.Matrix.LocRotScale.py

@@ -0,0 +1,5 @@
+# Compute local object transformation matrix:
+if obj.rotation_mode == 'QUATERNION':
+    matrix = mathutils.Matrix.LocRotScale(obj.location, obj.rotation_quaternion, obj.scale)
+else:
+    matrix = mathutils.Matrix.LocRotScale(obj.location, obj.rotation_euler, obj.scale)

doc/python_api/examples/mathutils.Matrix.py

@@ -14,10 +14,14 @@ mat_rot = mathutils.Matrix.Rotation(math.radians(45.0), 4, 'X')
 mat_out = mat_loc @ mat_rot @ mat_sca
 print(mat_out)
 
-# extract components back out of the matrix
+# extract components back out of the matrix as two vectors and a quaternion
 loc, rot, sca = mat_out.decompose()
 print(loc, rot, sca)
 
+# recombine extracted components
+mat_out2 = mathutils.Matrix.LocRotScale(loc, rot, sca)
+print(mat_out2)
+
 # it can also be useful to access components of a matrix directly
 mat = mathutils.Matrix()
 mat[0][0], mat[1][0], mat[2][0] = 0.0, 1.0, 2.0

doc/python_api/sphinx_doc_gen.py

@@ -545,6 +545,13 @@ def range_str(val):
 
 
 def example_extract_docstring(filepath):
+    '''
+    Return (text, line_no, line_no_has_content) where:
+    - ``text`` is the doc-string text.
+    - ``line_no`` is the line the doc-string text ends.
+    - ``line_no_has_content`` when False, this file only contains a doc-string.
+      There is no need to include the remainder.
+    '''
     file = open(filepath, "r", encoding="utf-8")
     line = file.readline()
     line_no = 0
@@ -553,7 +560,7 @@ def example_extract_docstring(filepath):
         line_no += 1
     else:
         file.close()
-        return "", 0, False
+        return "", 0, True
 
     for line in file:
         line_no += 1
@@ -947,7 +954,7 @@ def pymodule2sphinx(basepath, module_name, module, title, module_all_extra):
             # constant, not much fun we can do here except to list it.
             # TODO, figure out some way to document these!
             fw(".. data:: %s\n\n" % attribute)
-            write_indented_lines("   ", fw, "constant value %s" % repr(value), False)
+            write_indented_lines("   ", fw, "Constant value %s" % repr(value), False)
             fw("\n")
         else:
             BPY_LOGGER.debug("\tnot documenting %s.%s of %r type" % (module_name, attribute, value_type.__name__))
@@ -1239,7 +1246,7 @@ def pyrna_enum2sphinx(prop, use_empty_descriptions=False):
             "%s.\n" % (
                 identifier,
                 # Account for multi-line enum descriptions, allowing this to be a block of text.
-                indent(", ".join(escape_rst(val) for val in (name, description) if val) or "Undocumented", "  "),
+                indent(" -- ".join(escape_rst(val) for val in (name, description) if val) or "Undocumented", "  "),
             )
             for identifier, name, description in prop.enum_items
         ])

extern/audaspace/plugins/ffmpeg/FFMPEGWriter.cpp

@@ -75,6 +75,7 @@ void FFMPEGWriter::encode()
 	m_frame->nb_samples = m_input_samples;
 	m_frame->format = m_codecCtx->sample_fmt;
 	m_frame->channel_layout = m_codecCtx->channel_layout;
+	m_frame->channels = m_specs.channels;
 
 	if(avcodec_fill_audio_frame(m_frame, m_specs.channels, m_codecCtx->sample_fmt, reinterpret_cast<data_t*>(data), m_input_buffer.getSize(), 0) < 0)
 		AUD_THROW(FileException, "File couldn't be written, filling the audio frame failed with ffmpeg.");

extern/mantaflow/preprocessed/fileio/iovdb.cpp

@@ -29,10 +29,10 @@
 
 #if OPENVDB == 1
 #  include "openvdb/openvdb.h"
-#  include <openvdb/points/PointConversion.h>
-#  include <openvdb/points/PointCount.h>
-#  include <openvdb/tools/Clip.h>
-#  include <openvdb/tools/Dense.h>
+#  include "openvdb/points/PointConversion.h"
+#  include "openvdb/points/PointCount.h"
+#  include "openvdb/tools/Clip.h"
+#  include "openvdb/tools/Dense.h"
 #endif
 
 #define POSITION_NAME "P"
@@ -519,7 +519,7 @@ int writeObjectsVDB(const string &filename,
     }
   }
 
-  // Write only if the is at least one grid, optionally write with compression.
+  // Write only if there is at least one grid, optionally write with compression.
   if (gridsVDB.size()) {
     int vdb_flags = openvdb::io::COMPRESS_ACTIVE_MASK;
     switch (compression) {
@@ -534,7 +534,8 @@ int writeObjectsVDB(const string &filename,
       }
       case COMPRESSION_BLOSC: {
 #  if OPENVDB_BLOSC == 1
-        vdb_flags |= openvdb::io::COMPRESS_BLOSC;
+        // Cannot use |= here, causes segfault with blosc 1.5.0 (== recommended version)
+        vdb_flags = openvdb::io::COMPRESS_BLOSC;
 #  else
         debMsg("OpenVDB was built without Blosc support, using Zip compression instead", 1);
         vdb_flags |= openvdb::io::COMPRESS_ZIP;

extern/mantaflow/preprocessed/fluidsolver.h

@@ -384,6 +384,7 @@ class FluidSolver : public PbClass {
   GridStorage<Real> mGrids4dReal;
   GridStorage<Vec3> mGrids4dVec;
   GridStorage<Vec4> mGrids4dVec4;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/general.h

@@ -42,7 +42,7 @@ inline void updateQtGui(bool full, int frame, float time, const std::string &cur
 #  ifdef _DEBUG
 #    define DEBUG 1
 #  endif  // _DEBUG
-#endif  // DEBUG
+#endif    // DEBUG
 
 // Standard exception
 class Error : public std::exception {

extern/mantaflow/preprocessed/gitinfo.h

@@ -1,3 +1,3 @@
 
 
-#define MANTA_GIT_VERSION "commit 39b7a415721ecbf6643612a24e8eadd221aeb934"
+#define MANTA_GIT_VERSION "commit 9c505cd22e289b98c9aa717efba8ef3201c7e458"

extern/mantaflow/preprocessed/grid.h

@@ -389,6 +389,7 @@ class GridBase : public PbClass {
   Real mDx;
   bool m3D;  // precomputed Z shift: to ensure 2D compatibility, always use this instead of sx*sy !
   IndexInt mStrideZ;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/grid4d.h

@@ -326,6 +326,7 @@ class Grid4dBase : public PbClass {
   // precomputed Z,T shift: to ensure 2D compatibility, always use this instead of sx*sy !
   IndexInt mStrideZ;
   IndexInt mStrideT;
+
  public:
   PbArgs _args;
 }
@@ -950,6 +951,7 @@ template<class T> class Grid4d : public Grid4dBase {
 
  protected:
   T *mData;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/levelset.h

@@ -266,6 +266,7 @@ class LevelsetGrid : public Grid<Real> {
   }
 
   static Real invalidTimeValue();
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/mesh.h

@@ -796,6 +796,7 @@ class Mesh : public PbClass {
   std::vector<MeshDataImpl<int> *>
       mMdataInt;  //! indicate that mdata of this mesh is copied, and needs to be freed
   bool mFreeMdata;
+
  public:
   PbArgs _args;
 }
@@ -881,6 +882,7 @@ class MeshDataBase : public PbClass {
 
  protected:
   Mesh *mMesh;
+
  public:
   PbArgs _args;
 }
@@ -1645,6 +1647,7 @@ template<class T> class MeshDataImpl : public MeshDataBase {
   //! optionally , we might have an associated grid from which to grab new data
   Grid<T> *mpGridSource;  //! unfortunately , we need to distinguish mac vs regular vec3
   bool mGridSourceMAC;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/movingobs.h

@@ -154,6 +154,7 @@ class MovingObstacle : public PbClass {
   int mEmptyType;
   int mID;
   static int sIDcnt;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/noisefield.h

@@ -236,6 +236,7 @@ class WaveletNoiseField : public PbClass {
   static int randomSeed;
   // global reference count for noise tile
   static std::atomic<int> mNoiseReferenceCount;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/particle.h

@@ -205,6 +205,7 @@ class ParticleBase : public PbClass {
   //! custom seed for particle systems, used by plugins
   int mSeed;  //! fix global random seed storage, used mainly by functions in this class
   static int globalSeed;
+
  public:
   PbArgs _args;
 }
@@ -628,6 +629,7 @@ template<class S> class ParticleSystem : public ParticleBase {
   std::vector<S> mData;
   //! reduce storage , called by doCompress
   virtual void compress();
+
  public:
   PbArgs _args;
 }
@@ -918,6 +920,7 @@ class ParticleIndexSystem : public ParticleSystem<ParticleIndexData> {
       return -1;
     }
   };
+
  public:
   PbArgs _args;
 }
@@ -982,6 +985,7 @@ template<class DATA, class CON> class ConnectedParticleSystem : public ParticleS
  protected:
   std::vector<CON> mSegments;
   virtual void compress();
+
  public:
   PbArgs _args;
 }
@@ -1071,6 +1075,7 @@ class ParticleDataBase : public PbClass {
 
  protected:
   ParticleBase *mpParticleSys;
+
  public:
   PbArgs _args;
 }
@@ -1843,6 +1848,7 @@ template<class T> class ParticleDataImpl : public ParticleDataBase {
   //! optionally , we might have an associated grid from which to grab new data
   Grid<T> *mpGridSource;  //! unfortunately , we need to distinguish mac vs regular vec3
   bool mGridSourceMAC;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/plugin/meshplugins.cpp

@@ -234,10 +234,10 @@ void subdivideMesh(
       normalize(ne2);
 
       // Real thisArea = sqrMag(cross(-e2,e0));
-      // small angle approximation says sin(x) = arcsin(x) = x,
-      // arccos(x) = pi/2 - arcsin(x),
-      // cos(x) = dot(A,B),
-      // so angle is approximately 1 - dot(A,B).
+      //  small angle approximation says sin(x) = arcsin(x) = x,
+      //  arccos(x) = pi/2 - arcsin(x),
+      //  cos(x) = dot(A,B),
+      //  so angle is approximately 1 - dot(A,B).
       Real angle[3];
       angle[0] = 1.0 - dot(ne0, -ne2);
       angle[1] = 1.0 - dot(ne1, -ne0);

extern/mantaflow/preprocessed/plugin/secondaryparticles.cpp

@@ -2287,9 +2287,10 @@ struct knFlipComputePotentialTrappedAir : public KernelBase {
           const Vec3 &vj = scaleFromManta * v.getCentered(x, y, z);
           const Vec3 xij = xi - xj;
           const Vec3 vij = vi - vj;
-          Real h = !pot.is3D() ? 1.414 * radius :
-                                 1.732 * radius;  // estimate sqrt(2)*radius resp. sqrt(3)*radius
-                                                  // for h, due to squared resp. cubic neighbor area
+          Real h = !pot.is3D() ?
+                       1.414 * radius :
+                       1.732 * radius;  // estimate sqrt(2)*radius resp. sqrt(3)*radius for h, due
+                                        // to squared resp. cubic neighbor area
           vdiff += norm(vij) * (1 - dot(getNormalized(vij), getNormalized(xij))) *
                    (1 - norm(xij) / h);
         }

extern/mantaflow/preprocessed/shapes.h

@@ -269,6 +269,7 @@ class Shape : public PbClass {
 
  protected:
   GridType mType;
+
  public:
   PbArgs _args;
 }
@@ -319,6 +320,7 @@ class NullShape : public Shape {
   {
     gridSetConst<Real>(phi, 1000.0f);
   }
+
  public:
   PbArgs _args;
 }
@@ -394,6 +396,7 @@ class Box : public Shape {
 
  protected:
   Vec3 mP0, mP1;
+
  public:
   PbArgs _args;
 }
@@ -455,6 +458,7 @@ class Sphere : public Shape {
  protected:
   Vec3 mCenter, mScale;
   Real mRadius;
+
  public:
   PbArgs _args;
 }
@@ -579,6 +583,7 @@ class Cylinder : public Shape {
  protected:
   Vec3 mCenter, mZDir;
   Real mRadius, mZ;
+
  public:
   PbArgs _args;
 }
@@ -655,6 +660,7 @@ class Slope : public Shape {
   Real mAnglexy, mAngleyz;
   Real mOrigin;
   Vec3 mGs;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/turbulencepart.h

@@ -199,6 +199,7 @@ class TurbulenceParticleSystem : public ParticleSystem<TurbulenceParticleData> {
 
  private:
   WaveletNoiseField &noise;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/vortexpart.h

@@ -127,6 +127,7 @@ class VortexParticleSystem : public ParticleSystem<VortexParticleData> {
   }
 
   virtual ParticleBase *clone();
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/vortexsheet.h

@@ -240,6 +240,7 @@ class VortexSheetMesh : public Mesh {
   VorticityChannel mVorticity;
   TexCoord3Channel mTex1, mTex2;
   TurbulenceChannel mTurb;
+
  public:
   PbArgs _args;
 }

intern/clog/clog.c

@@ -322,7 +322,9 @@ static bool clg_ctx_filter_check(CLogContext *ctx, const char *identifier)
       if (flt->match[0] == '*' && flt->match[len - 1] == '*') {
         char *match = MEM_callocN(sizeof(char) * len - 1, __func__);
         memcpy(match, flt->match + 1, len - 2);
-        if (strstr(identifier, match) != NULL) {
+        const bool success = (strstr(identifier, match) != NULL);
+        MEM_freeN(match);
+        if (success) {
           return (bool)i;
         }
       }

intern/cycles/blender/addon/engine.py

@@ -19,16 +19,16 @@ from __future__ import annotations
 
 
 def _is_using_buggy_driver():
-    import bgl
+    import gpu
     # We need to be conservative here because in multi-GPU systems display card
     # might be quite old, but others one might be just good.
     #
     # So We shouldn't disable possible good dedicated cards just because display
     # card seems weak. And instead we only blacklist configurations which are
     # proven to cause problems.
-    if bgl.glGetString(bgl.GL_VENDOR) == "ATI Technologies Inc.":
+    if gpu.platform.vendor_get() == "ATI Technologies Inc.":
         import re
-        version = bgl.glGetString(bgl.GL_VERSION)
+        version = gpu.platform.version_get()
         if version.endswith("Compatibility Profile Context"):
             # Old HD 4xxx and 5xxx series drivers did not have driver version
             # in the version string, but those cards do not quite work and

intern/cycles/blender/addon/ui.py

@@ -725,7 +725,7 @@ class CYCLES_RENDER_PT_performance_tiles(CyclesButtonsPanel, Panel):
         col.prop(cscene, "tile_order", text="Order")
 
         sub = col.column()
-        sub.active = not rd.use_save_buffers
+        sub.active = not rd.use_save_buffers and not cscene.use_adaptive_sampling
         sub.prop(cscene, "use_progressive_refine")
 
 

intern/cycles/blender/blender_camera.cpp

@@ -83,6 +83,8 @@ struct BlenderCamera {
   BoundBox2D pano_viewplane;
   BoundBox2D viewport_camera_border;
 
+  float passepartout_alpha;
+
   Transform matrix;
 
   float offscreen_dicing_scale;
@@ -125,6 +127,7 @@ static void blender_camera_init(BlenderCamera *bcam, BL::RenderSettings &b_rende
   bcam->pano_viewplane.top = 1.0f;
   bcam->viewport_camera_border.right = 1.0f;
   bcam->viewport_camera_border.top = 1.0f;
+  bcam->passepartout_alpha = 0.5f;
   bcam->offscreen_dicing_scale = 1.0f;
   bcam->matrix = transform_identity();
 
@@ -212,6 +215,8 @@ static void blender_camera_from_object(BlenderCamera *bcam,
 
     bcam->lens = b_camera.lens();
 
+    bcam->passepartout_alpha = b_camera.show_passepartout() ? b_camera.passepartout_alpha() : 0.0f;
+
     if (b_camera.dof().use_dof()) {
       /* allow f/stop number to change aperture_size but still
        * give manual control over aperture radius */
@@ -834,15 +839,19 @@ static void blender_camera_border(BlenderCamera *bcam,
                                full_border,
                                &bcam->viewport_camera_border);
 
-  if (!b_render.use_border()) {
+  if (b_render.use_border()) {
+    bcam->border.left = b_render.border_min_x();
+    bcam->border.right = b_render.border_max_x();
+    bcam->border.bottom = b_render.border_min_y();
+    bcam->border.top = b_render.border_max_y();
+  }
+  else if (bcam->passepartout_alpha == 1.0f) {
+    bcam->border = full_border;
+  }
+  else {
     return;
   }
 
-  bcam->border.left = b_render.border_min_x();
-  bcam->border.right = b_render.border_max_x();
-  bcam->border.bottom = b_render.border_min_y();
-  bcam->border.top = b_render.border_max_y();
-
   /* Determine viewport subset matching camera border. */
   blender_camera_border_subset(b_engine,
                                b_render,
@@ -885,8 +894,7 @@ void BlenderSync::sync_view(BL::SpaceView3D &b_v3d,
   }
 }
 
-BufferParams BlenderSync::get_buffer_params(BL::RenderSettings &b_render,
-                                            BL::SpaceView3D &b_v3d,
+BufferParams BlenderSync::get_buffer_params(BL::SpaceView3D &b_v3d,
                                             BL::RegionView3D &b_rv3d,
                                             Camera *cam,
                                             int width,
@@ -902,7 +910,8 @@ BufferParams BlenderSync::get_buffer_params(BL::RenderSettings &b_render,
   if (b_v3d && b_rv3d && b_rv3d.view_perspective() != BL::RegionView3D::view_perspective_CAMERA)
     use_border = b_v3d.use_render_border();
   else
-    use_border = b_render.use_border();
+    /* the camera can always have a passepartout */
+    use_border = true;
 
   if (use_border) {
     /* border render */

intern/cycles/blender/blender_object.cpp

@@ -261,10 +261,8 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
   }
 
   /* test if we need to sync */
-  bool object_updated = false;
-
-  if (object_map.add_or_update(&object, b_ob, b_parent, key))
-    object_updated = true;
+  bool object_updated = object_map.add_or_update(&object, b_ob, b_parent, key) ||
+                        (tfm != object->get_tfm());
 
   /* mesh sync */
   /* b_ob is owned by the iterator and will go out of scope at the end of the block.
@@ -313,8 +311,7 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
    * transform comparison should not be needed, but duplis don't work perfect
    * in the depsgraph and may not signal changes, so this is a workaround */
   if (object->is_modified() || object_updated ||
-      (object->get_geometry() && object->get_geometry()->is_modified()) ||
-      tfm != object->get_tfm()) {
+      (object->get_geometry() && object->get_geometry()->is_modified())) {
     object->name = b_ob.name().c_str();
     object->set_pass_id(b_ob.pass_index());
     object->set_color(get_float3(b_ob.color()));

intern/cycles/blender/blender_session.cpp

@@ -155,7 +155,7 @@ void BlenderSession::create_session()
 
   /* set buffer parameters */
   BufferParams buffer_params = BlenderSync::get_buffer_params(
-      b_render, b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
+      b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
   session->reset(buffer_params, session_params.samples);
 
   b_engine.use_highlight_tiles(session_params.progressive_refine == false);
@@ -242,8 +242,7 @@ void BlenderSession::reset_session(BL::BlendData &b_data, BL::Depsgraph &b_depsg
 
   BL::SpaceView3D b_null_space_view3d(PointerRNA_NULL);
   BL::RegionView3D b_null_region_view3d(PointerRNA_NULL);
-  BufferParams buffer_params = BlenderSync::get_buffer_params(b_render,
-                                                              b_null_space_view3d,
+  BufferParams buffer_params = BlenderSync::get_buffer_params(b_null_space_view3d,
                                                               b_null_region_view3d,
                                                               scene->camera,
                                                               width,
@@ -486,7 +485,7 @@ void BlenderSession::render(BL::Depsgraph &b_depsgraph_)
   SessionParams session_params = BlenderSync::get_session_params(
       b_engine, b_userpref, b_scene, background, b_view_layer);
   BufferParams buffer_params = BlenderSync::get_buffer_params(
-      b_render, b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
+      b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
 
   /* temporary render result to find needed passes and views */
   BL::RenderResult b_rr = begin_render_result(
@@ -810,7 +809,7 @@ void BlenderSession::synchronize(BL::Depsgraph &b_depsgraph_)
 
   /* get buffer parameters */
   BufferParams buffer_params = BlenderSync::get_buffer_params(
-      b_render, b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
+      b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
 
   if (!buffer_params.denoising_data_pass) {
     session_params.denoising.use = false;
@@ -889,7 +888,7 @@ bool BlenderSession::draw(int w, int h)
       SessionParams session_params = BlenderSync::get_session_params(
           b_engine, b_userpref, b_scene, background);
       BufferParams buffer_params = BlenderSync::get_buffer_params(
-          b_render, b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
+          b_v3d, b_rv3d, scene->camera, width, height, session_params.denoising.use);
       bool session_pause = BlenderSync::get_session_pause(b_scene, background);
 
       if (session_pause == false) {
@@ -907,7 +906,7 @@ bool BlenderSession::draw(int w, int h)
 
   /* draw */
   BufferParams buffer_params = BlenderSync::get_buffer_params(
-      b_render, b_v3d, b_rv3d, scene->camera, width, height, session->params.denoising.use);
+      b_v3d, b_rv3d, scene->camera, width, height, session->params.denoising.use);
   DeviceDrawParams draw_params;
 
   if (session->params.display_buffer_linear) {

intern/cycles/blender/blender_shader.cpp

@@ -1373,7 +1373,7 @@ void BlenderSync::sync_world(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d,
   BlenderViewportParameters new_viewport_parameters(b_v3d);
 
   if (world_recalc || update_all || b_world.ptr.data != world_map ||
-      viewport_parameters.modified(new_viewport_parameters)) {
+      viewport_parameters.shader_modified(new_viewport_parameters)) {
     Shader *shader = scene->default_background;
     ShaderGraph *graph = new ShaderGraph();
 
@@ -1501,8 +1501,8 @@ void BlenderSync::sync_world(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d,
     background->set_transparent_roughness_threshold(0.0f);
   }
 
-  background->set_use_shader(view_layer.use_background_shader |
-                             viewport_parameters.custom_viewport_parameters());
+  background->set_use_shader(view_layer.use_background_shader ||
+                             viewport_parameters.use_custom_shader());
   background->set_use_ao(background->get_use_ao() && view_layer.use_background_ao);
 
   background->tag_update(scene);
@@ -1553,13 +1553,9 @@ void BlenderSync::sync_lights(BL::Depsgraph &b_depsgraph, bool update_all)
 void BlenderSync::sync_shaders(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d)
 {
   /* for auto refresh images */
-  bool auto_refresh_update = false;
-
-  if (preview) {
-    ImageManager *image_manager = scene->image_manager;
-    int frame = b_scene.frame_current();
-    auto_refresh_update = image_manager->set_animation_frame_update(frame);
-  }
+  ImageManager *image_manager = scene->image_manager;
+  const int frame = b_scene.frame_current();
+  const bool auto_refresh_update = image_manager->set_animation_frame_update(frame);
 
   shader_map.pre_sync();
 

intern/cycles/blender/blender_sync.cpp

@@ -69,7 +69,8 @@ BlenderSync::BlenderSync(BL::RenderEngine &b_engine,
       experimental(false),
       dicing_rate(1.0f),
       max_subdivisions(12),
-      progress(progress)
+      progress(progress),
+      has_updates_(true)
 {
   PointerRNA cscene = RNA_pointer_get(&b_scene.ptr, "cycles");
   dicing_rate = preview ? RNA_float_get(&cscene, "preview_dicing_rate") :
@@ -84,7 +85,9 @@ BlenderSync::~BlenderSync()
 void BlenderSync::reset(BL::BlendData &b_data, BL::Scene &b_scene)
 {
   /* Update data and scene pointers in case they change in session reset,
-   * for example after undo. */
+   * for example after undo.
+   * Note that we do not modify the `has_updates_` flag here because the sync
+   * reset is also used during viewport navigation. */
   this->b_data = b_data;
   this->b_scene = b_scene;
 }
@@ -117,6 +120,8 @@ void BlenderSync::sync_recalc(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d
     }
 
     if (dicing_prop_changed) {
+      has_updates_ = true;
+
       for (const pair<const GeometryKey, Geometry *> &iter : geometry_map.key_to_scene_data()) {
         Geometry *geom = iter.second;
         if (geom->is_mesh()) {
@@ -133,6 +138,12 @@ void BlenderSync::sync_recalc(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d
 
   /* Iterate over all IDs in this depsgraph. */
   for (BL::DepsgraphUpdate &b_update : b_depsgraph.updates) {
+    /* TODO(sergey): Can do more selective filter here. For example, ignore changes made to
+     * screen datablock. Note that sync_data() needs to be called after object deletion, and
+     * currently this is ensured by the scene ID tagged for update, which sets the `has_updates_`
+     * flag. */
+    has_updates_ = true;
+
     BL::ID b_id(b_update.id());
 
     /* Material */
@@ -213,9 +224,13 @@ void BlenderSync::sync_recalc(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d
 
   if (b_v3d) {
     BlenderViewportParameters new_viewport_parameters(b_v3d);
-    if (viewport_parameters.modified(new_viewport_parameters)) {
+
+    if (viewport_parameters.shader_modified(new_viewport_parameters)) {
       world_recalc = true;
+      has_updates_ = true;
     }
+
+    has_updates_ |= viewport_parameters.modified(new_viewport_parameters);
   }
 }
 
@@ -227,11 +242,15 @@ void BlenderSync::sync_data(BL::RenderSettings &b_render,
                             int height,
                             void **python_thread_state)
 {
+  if (!has_updates_) {
+    return;
+  }
+
   scoped_timer timer;
 
   BL::ViewLayer b_view_layer = b_depsgraph.view_layer_eval();
 
-  sync_view_layer(b_v3d, b_view_layer);
+  sync_view_layer(b_view_layer);
   sync_integrator();
   sync_film(b_v3d);
   sync_shaders(b_depsgraph, b_v3d);
@@ -254,6 +273,8 @@ void BlenderSync::sync_data(BL::RenderSettings &b_render,
   free_data_after_sync(b_depsgraph);
 
   VLOG(1) << "Total time spent synchronizing data: " << timer.get_time();
+
+  has_updates_ = false;
 }
 
 /* Integrator */
@@ -424,7 +445,7 @@ void BlenderSync::sync_film(BL::SpaceView3D &b_v3d)
 
 /* Render Layer */
 
-void BlenderSync::sync_view_layer(BL::SpaceView3D & /*b_v3d*/, BL::ViewLayer &b_view_layer)
+void BlenderSync::sync_view_layer(BL::ViewLayer &b_view_layer)
 {
   view_layer.name = b_view_layer.name();
 
@@ -875,6 +896,9 @@ SessionParams BlenderSync::get_session_params(BL::RenderEngine &b_engine,
   /* Clamp samples. */
   params.samples = min(params.samples, Integrator::MAX_SAMPLES);
 
+  /* Adaptive sampling. */
+  params.adaptive_sampling = RNA_boolean_get(&cscene, "use_adaptive_sampling");
+
   /* tiles */
   const bool is_cpu = (params.device.type == DEVICE_CPU);
   if (!is_cpu && !background) {
@@ -927,7 +951,7 @@ SessionParams BlenderSync::get_session_params(BL::RenderEngine &b_engine,
   BL::RenderSettings b_r = b_scene.render();
   params.progressive_refine = b_engine.is_preview() ||
                               get_boolean(cscene, "use_progressive_refine");
-  if (b_r.use_save_buffers())
+  if (b_r.use_save_buffers() || params.adaptive_sampling)
     params.progressive_refine = false;
 
   if (background) {
@@ -963,8 +987,6 @@ SessionParams BlenderSync::get_session_params(BL::RenderEngine &b_engine,
   params.use_profiling = params.device.has_profiling && !b_engine.is_preview() && background &&
                          BlenderSession::print_render_stats;
 
-  params.adaptive_sampling = RNA_boolean_get(&cscene, "use_adaptive_sampling");
-
   return params;
 }
 

intern/cycles/blender/blender_sync.h

@@ -73,7 +73,7 @@ class BlenderSync {
                  int width,
                  int height,
                  void **python_thread_state);
-  void sync_view_layer(BL::SpaceView3D &b_v3d, BL::ViewLayer &b_view_layer);
+  void sync_view_layer(BL::ViewLayer &b_view_layer);
   vector<Pass> sync_render_passes(BL::Scene &b_scene,
                                   BL::RenderLayer &b_render_layer,
                                   BL::ViewLayer &b_view_layer,
@@ -104,8 +104,7 @@ class BlenderSync {
       bool background,
       BL::ViewLayer b_view_layer = BL::ViewLayer(PointerRNA_NULL));
   static bool get_session_pause(BL::Scene &b_scene, bool background);
-  static BufferParams get_buffer_params(BL::RenderSettings &b_render,
-                                        BL::SpaceView3D &b_v3d,
+  static BufferParams get_buffer_params(BL::SpaceView3D &b_v3d,
                                         BL::RegionView3D &b_rv3d,
                                         Camera *cam,
                                         int width,
@@ -264,6 +263,12 @@ class BlenderSync {
   } view_layer;
 
   Progress &progress;
+
+ protected:
+  /* Indicates that `sync_recalc()` detected changes in the scene.
+   * If this flag is false then the data is considered to be up-to-date and will not be
+   * synchronized at all. */
+  bool has_updates_ = true;
 };
 
 CCL_NAMESPACE_END

intern/cycles/blender/blender_viewport.cpp

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #include "blender_viewport.h"
 
 #include "blender_util.h"
@@ -25,29 +26,39 @@ BlenderViewportParameters::BlenderViewportParameters()
       studiolight_rotate_z(0.0f),
       studiolight_intensity(1.0f),
       studiolight_background_alpha(1.0f),
-      studiolight_path(ustring())
+      display_pass(PASS_COMBINED)
 {
 }
 
 BlenderViewportParameters::BlenderViewportParameters(BL::SpaceView3D &b_v3d)
     : BlenderViewportParameters()
 {
-  /* We only copy the parameters if we are in look dev mode. otherwise
+  if (!b_v3d) {
+    return;
+  }
+
+  BL::View3DShading shading = b_v3d.shading();
+  PointerRNA cshading = RNA_pointer_get(&shading.ptr, "cycles");
+
+  /* We only copy the shading parameters if we are in look dev mode. otherwise
    * defaults are being used. These defaults mimic normal render settings */
-  if (b_v3d && b_v3d.shading().type() == BL::View3DShading::type_RENDERED) {
-    use_scene_world = b_v3d.shading().use_scene_world_render();
-    use_scene_lights = b_v3d.shading().use_scene_lights_render();
+  if (shading.type() == BL::View3DShading::type_RENDERED) {
+    use_scene_world = shading.use_scene_world_render();
+    use_scene_lights = shading.use_scene_lights_render();
+
     if (!use_scene_world) {
-      studiolight_rotate_z = b_v3d.shading().studiolight_rotate_z();
-      studiolight_intensity = b_v3d.shading().studiolight_intensity();
-      studiolight_background_alpha = b_v3d.shading().studiolight_background_alpha();
-      studiolight_path = b_v3d.shading().selected_studio_light().path();
+      studiolight_rotate_z = shading.studiolight_rotate_z();
+      studiolight_intensity = shading.studiolight_intensity();
+      studiolight_background_alpha = shading.studiolight_background_alpha();
+      studiolight_path = shading.selected_studio_light().path();
     }
   }
+
+  /* Film. */
+  display_pass = (PassType)get_enum(cshading, "render_pass", -1, -1);
 }
 
-/* Check if two instances are different. */
-const bool BlenderViewportParameters::modified(const BlenderViewportParameters &other) const
+bool BlenderViewportParameters::shader_modified(const BlenderViewportParameters &other) const
 {
   return use_scene_world != other.use_scene_world || use_scene_lights != other.use_scene_lights ||
          studiolight_rotate_z != other.studiolight_rotate_z ||
@@ -56,26 +67,26 @@ const bool BlenderViewportParameters::modified(const BlenderViewportParameters &
          studiolight_path != other.studiolight_path;
 }
 
-const bool BlenderViewportParameters::custom_viewport_parameters() const
+bool BlenderViewportParameters::film_modified(const BlenderViewportParameters &other) const
 {
-  return !(use_scene_world && use_scene_lights);
+  return display_pass != other.display_pass;
 }
 
-PassType BlenderViewportParameters::get_viewport_display_render_pass(BL::SpaceView3D &b_v3d)
+bool BlenderViewportParameters::modified(const BlenderViewportParameters &other) const
 {
-  PassType display_pass = PASS_NONE;
-  if (b_v3d) {
-    BL::View3DShading b_view3dshading = b_v3d.shading();
-    PointerRNA cshading = RNA_pointer_get(&b_view3dshading.ptr, "cycles");
-    display_pass = (PassType)get_enum(cshading, "render_pass", -1, -1);
-  }
-  return display_pass;
+  return shader_modified(other) || film_modified(other);
+}
+
+bool BlenderViewportParameters::use_custom_shader() const
+{
+  return !(use_scene_world && use_scene_lights);
 }
 
 PassType update_viewport_display_passes(BL::SpaceView3D &b_v3d, vector<Pass> &passes)
 {
   if (b_v3d) {
-    PassType display_pass = BlenderViewportParameters::get_viewport_display_render_pass(b_v3d);
+    const BlenderViewportParameters viewport_parameters(b_v3d);
+    const PassType display_pass = viewport_parameters.display_pass;
 
     passes.clear();
     Pass::add(display_pass, passes);

intern/cycles/blender/blender_viewport.h

@@ -18,17 +18,18 @@
 #define __BLENDER_VIEWPORT_H__
 
 #include "MEM_guardedalloc.h"
+
 #include "RNA_access.h"
 #include "RNA_blender_cpp.h"
 #include "RNA_types.h"
 
 #include "render/film.h"
-#include "util/util_param.h"
 
 CCL_NAMESPACE_BEGIN
 
 class BlenderViewportParameters {
- private:
+ public:
+  /* Shader. */
   bool use_scene_world;
   bool use_scene_lights;
   float studiolight_rotate_z;
@@ -36,17 +37,24 @@ class BlenderViewportParameters {
   float studiolight_background_alpha;
   ustring studiolight_path;
 
+  /* Film. */
+  PassType display_pass;
+
   BlenderViewportParameters();
-  BlenderViewportParameters(BL::SpaceView3D &b_v3d);
+  explicit BlenderViewportParameters(BL::SpaceView3D &b_v3d);
 
-  const bool modified(const BlenderViewportParameters &other) const;
-  const bool custom_viewport_parameters() const;
-  friend class BlenderSync;
+  /* Check whether any of shading related settings are different from the given parameters. */
+  bool shader_modified(const BlenderViewportParameters &other) const;
 
- public:
-  /* Retrieve the render pass that needs to be displayed on the given `SpaceView3D`
-   * When the `b_v3d` parameter is not given `PASS_NONE` will be returned. */
-  static PassType get_viewport_display_render_pass(BL::SpaceView3D &b_v3d);
+  /* Check whether any of film related settings are different from the given parameters. */
+  bool film_modified(const BlenderViewportParameters &other) const;
+
+  /* Check whether any of settings are different from the given parameters. */
+  bool modified(const BlenderViewportParameters &other) const;
+
+  /* Returns truth when a custom shader defined by the viewport is to be used instead of the
+   * regular background shader or scene light. */
+  bool use_custom_shader() const;
 };
 
 PassType update_viewport_display_passes(BL::SpaceView3D &b_v3d, vector<Pass> &passes);

intern/cycles/bvh/bvh_optix.cpp

@@ -17,6 +17,8 @@
 
 #ifdef WITH_OPTIX
 
+#  include "device/device.h"
+
 #  include "bvh/bvh_optix.h"
 
 CCL_NAMESPACE_BEGIN
@@ -26,6 +28,7 @@ BVHOptiX::BVHOptiX(const BVHParams &params_,
                    const vector<Object *> &objects_,
                    Device *device)
     : BVH(params_, geometry_, objects_),
+      device(device),
       traversable_handle(0),
       as_data(device, params_.top_level ? "optix tlas" : "optix blas", false),
       motion_transform_data(device, "optix motion transform", false)
@@ -34,7 +37,9 @@ BVHOptiX::BVHOptiX(const BVHParams &params_,
 
 BVHOptiX::~BVHOptiX()
 {
-  // Acceleration structure memory is freed via the 'as_data' destructor
+  // Acceleration structure memory is delayed freed on device, since deleting the
+  // BVH may happen while still being used for rendering.
+  device->release_optix_bvh(this);
 }
 
 CCL_NAMESPACE_END

intern/cycles/bvh/bvh_optix.h

@@ -28,6 +28,7 @@ CCL_NAMESPACE_BEGIN
 
 class BVHOptiX : public BVH {
  public:
+  Device *device;
   uint64_t traversable_handle;
   device_only_memory<char> as_data;
   device_only_memory<char> motion_transform_data;

intern/cycles/device/device.h

@@ -61,7 +61,6 @@ enum DeviceTypeMask {
 };
 
 enum DeviceKernelStatus {
-  DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL = 0,
   DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE,
   DEVICE_KERNEL_USING_FEATURE_KERNEL,
   DEVICE_KERNEL_FEATURE_KERNEL_INVALID,
@@ -427,6 +426,9 @@ class Device {
   /* acceleration structure building */
   virtual void build_bvh(BVH *bvh, Progress &progress, bool refit);
 
+  /* OptiX specific destructor. */
+  virtual void release_optix_bvh(BVH * /*bvh*/){};
+
 #ifdef WITH_NETWORK
   /* networking */
   void server_run();

intern/cycles/device/device_memory.cpp

@@ -35,10 +35,54 @@ device_memory::device_memory(Device *device, const char *name, MemoryType type)
       device_pointer(0),
       host_pointer(0),
       shared_pointer(0),
-      shared_counter(0)
+      shared_counter(0),
+      original_device_ptr(0),
+      original_device_size(0),
+      original_device(0),
+      need_realloc_(false),
+      modified(false)
 {
 }
 
+device_memory::device_memory(device_memory &&other) noexcept
+    : data_type(other.data_type),
+      data_elements(other.data_elements),
+      data_size(other.data_size),
+      device_size(other.device_size),
+      data_width(other.data_width),
+      data_height(other.data_height),
+      data_depth(other.data_depth),
+      type(other.type),
+      name(other.name),
+      device(other.device),
+      device_pointer(other.device_pointer),
+      host_pointer(other.host_pointer),
+      shared_pointer(other.shared_pointer),
+      shared_counter(other.shared_counter),
+      original_device_ptr(other.original_device_ptr),
+      original_device_size(other.original_device_size),
+      original_device(other.original_device),
+      need_realloc_(other.need_realloc_),
+      modified(other.modified)
+{
+  other.data_elements = 0;
+  other.data_size = 0;
+  other.device_size = 0;
+  other.data_width = 0;
+  other.data_height = 0;
+  other.data_depth = 0;
+  other.device = 0;
+  other.device_pointer = 0;
+  other.host_pointer = 0;
+  other.shared_pointer = 0;
+  other.shared_counter = 0;
+  other.original_device_ptr = 0;
+  other.original_device_size = 0;
+  other.original_device = 0;
+  other.need_realloc_ = false;
+  other.modified = false;
+}
+
 device_memory::~device_memory()
 {
   assert(shared_pointer == 0);

intern/cycles/device/device_memory.h

@@ -238,6 +238,7 @@ class device_memory {
 
   /* Only create through subclasses. */
   device_memory(Device *device, const char *name, MemoryType type);
+  device_memory(device_memory &&other) noexcept;
 
   /* No copying allowed. */
   device_memory(const device_memory &) = delete;
@@ -277,6 +278,10 @@ template<typename T> class device_only_memory : public device_memory {
     data_elements = max(device_type_traits<T>::num_elements, 1);
   }
 
+  device_only_memory(device_only_memory &&other) noexcept : device_memory(std::move(other))
+  {
+  }
+
   virtual ~device_only_memory()
   {
     free();

intern/cycles/device/device_multi.cpp

@@ -232,10 +232,6 @@ class MultiDevice : public Device {
     foreach (SubDevice &sub, devices) {
       DeviceKernelStatus subresult = sub.device->get_active_kernel_switch_state();
       switch (subresult) {
-        case DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL:
-          result = subresult;
-          break;
-
         case DEVICE_KERNEL_FEATURE_KERNEL_INVALID:
         case DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE:
           return subresult;

intern/cycles/device/device_optix.cpp

@@ -193,6 +193,9 @@ class OptiXDevice : public CUDADevice {
   device_only_memory<unsigned char> denoiser_state;
   int denoiser_input_passes = 0;
 
+  vector<device_only_memory<char>> delayed_free_bvh_memory;
+  thread_mutex delayed_free_bvh_mutex;
+
  public:
   OptiXDevice(DeviceInfo &info_, Stats &stats_, Profiler &profiler_, bool background_)
       : CUDADevice(info_, stats_, profiler_, background_),
@@ -258,6 +261,8 @@ class OptiXDevice : public CUDADevice {
     // Make CUDA context current
     const CUDAContextScope scope(cuContext);
 
+    free_bvh_memory_delayed();
+
     sbt_data.free();
     texture_info.free();
     launch_params.free();
@@ -721,7 +726,11 @@ class OptiXDevice : public CUDADevice {
       }
     }
     else if (task.type == DeviceTask::SHADER) {
-      launch_shader_eval(task, thread_index);
+      // CUDA kernels are used when doing baking
+      if (optix_module == NULL)
+        CUDADevice::shader(task);
+      else
+        launch_shader_eval(task, thread_index);
     }
     else if (task.type == DeviceTask::DENOISE_BUFFER) {
       // Set up a single tile that covers the whole task and denoise it
@@ -1297,6 +1306,8 @@ class OptiXDevice : public CUDADevice {
       return;
     }
 
+    free_bvh_memory_delayed();
+
     BVHOptiX *const bvh_optix = static_cast<BVHOptiX *>(bvh);
 
     progress.set_substatus("Building OptiX acceleration structure");
@@ -1767,6 +1778,24 @@ class OptiXDevice : public CUDADevice {
     }
   }
 
+  void release_optix_bvh(BVH *bvh) override
+  {
+    thread_scoped_lock lock(delayed_free_bvh_mutex);
+    /* Do delayed free of BVH memory, since geometry holding BVH might be deleted
+     * while GPU is still rendering. */
+    BVHOptiX *const bvh_optix = static_cast<BVHOptiX *>(bvh);
+
+    delayed_free_bvh_memory.emplace_back(std::move(bvh_optix->as_data));
+    delayed_free_bvh_memory.emplace_back(std::move(bvh_optix->motion_transform_data));
+    bvh_optix->traversable_handle = 0;
+  }
+
+  void free_bvh_memory_delayed()
+  {
+    thread_scoped_lock lock(delayed_free_bvh_mutex);
+    delayed_free_bvh_memory.free_memory();
+  }
+
   void const_copy_to(const char *name, void *host, size_t size) override
   {
     // Set constant memory for CUDA module

intern/cycles/device/opencl/device_opencl.h

@@ -269,7 +269,6 @@ class OpenCLDevice : public Device {
   cl_device_id cdDevice;
   cl_int ciErr;
   int device_num;
-  bool use_preview_kernels;
 
   class OpenCLProgram {
    public:
@@ -369,8 +368,7 @@ class OpenCLDevice : public Device {
     /* Load the kernels and put the created kernels in the given
      * `programs` parameter. */
     void load_kernels(vector<OpenCLProgram *> &programs,
-                      const DeviceRequestedFeatures &requested_features,
-                      bool is_preview = false);
+                      const DeviceRequestedFeatures &requested_features);
   };
 
   DeviceSplitKernel *split_kernel;
@@ -382,7 +380,6 @@ class OpenCLDevice : public Device {
   OpenCLProgram denoising_program;
 
   OpenCLSplitPrograms kernel_programs;
-  OpenCLSplitPrograms preview_programs;
 
   typedef map<string, device_vector<uchar> *> ConstMemMap;
   typedef map<string, device_ptr> MemMap;
@@ -412,7 +409,6 @@ class OpenCLDevice : public Device {
   string device_md5_hash(string kernel_custom_build_options = "");
   bool load_kernels(const DeviceRequestedFeatures &requested_features);
   void load_required_kernels(const DeviceRequestedFeatures &requested_features);
-  void load_preview_kernels();
 
   bool wait_for_availability(const DeviceRequestedFeatures &requested_features);
   DeviceKernelStatus get_active_kernel_switch_state();
@@ -422,8 +418,7 @@ class OpenCLDevice : public Device {
   /* Get the program file name to compile (*.cl) for the given kernel */
   const string get_opencl_program_filename(const string &kernel_name);
   string get_build_options(const DeviceRequestedFeatures &requested_features,
-                           const string &opencl_program_name,
-                           bool preview_kernel = false);
+                           const string &opencl_program_name);
   /* Enable the default features to reduce recompilation events */
   void enable_default_features(DeviceRequestedFeatures &features);
 

intern/cycles/device/opencl/device_opencl_impl.cpp

@@ -107,8 +107,7 @@ void OpenCLDevice::enable_default_features(DeviceRequestedFeatures &features)
 }
 
 string OpenCLDevice::get_build_options(const DeviceRequestedFeatures &requested_features,
-                                       const string &opencl_program_name,
-                                       bool preview_kernel)
+                                       const string &opencl_program_name)
 {
   /* first check for non-split kernel programs */
   if (opencl_program_name == "base" || opencl_program_name == "denoising") {
@@ -185,13 +184,7 @@ string OpenCLDevice::get_build_options(const DeviceRequestedFeatures &requested_
   enable_default_features(nofeatures);
 
   /* Add program specific optimized compile directives */
-  if (preview_kernel) {
-    DeviceRequestedFeatures preview_features;
-    preview_features.use_hair = true;
-    build_options += "-D__KERNEL_AO_PREVIEW__ ";
-    build_options += preview_features.get_build_options();
-  }
-  else if (opencl_program_name == "split_do_volume" && !requested_features.use_volume) {
+  if (opencl_program_name == "split_do_volume" && !requested_features.use_volume) {
     build_options += nofeatures.get_build_options();
   }
   else {
@@ -238,9 +231,7 @@ OpenCLDevice::OpenCLSplitPrograms::~OpenCLSplitPrograms()
 }
 
 void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
-    vector<OpenCLProgram *> &programs,
-    const DeviceRequestedFeatures &requested_features,
-    bool is_preview)
+    vector<OpenCLProgram *> &programs, const DeviceRequestedFeatures &requested_features)
 {
   if (!requested_features.use_baking) {
 #  define ADD_SPLIT_KERNEL_BUNDLE_PROGRAM(kernel_name) \
@@ -251,7 +242,7 @@ void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
         device, \
         program_name_##kernel_name, \
         "kernel_" #kernel_name ".cl", \
-        device->get_build_options(requested_features, program_name_##kernel_name, is_preview)); \
+        device->get_build_options(requested_features, program_name_##kernel_name)); \
     program_##kernel_name.add_kernel(ustring("path_trace_" #kernel_name)); \
     programs.push_back(&program_##kernel_name);
 
@@ -259,7 +250,7 @@ void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
     ADD_SPLIT_KERNEL_PROGRAM(subsurface_scatter);
     ADD_SPLIT_KERNEL_PROGRAM(direct_lighting);
     ADD_SPLIT_KERNEL_PROGRAM(indirect_background);
-    if (requested_features.use_volume || is_preview) {
+    if (requested_features.use_volume) {
       ADD_SPLIT_KERNEL_PROGRAM(do_volume);
     }
     ADD_SPLIT_KERNEL_PROGRAM(shader_eval);
@@ -274,7 +265,7 @@ void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
         device,
         "split_bundle",
         "kernel_split_bundle.cl",
-        device->get_build_options(requested_features, "split_bundle", is_preview));
+        device->get_build_options(requested_features, "split_bundle"));
 
     ADD_SPLIT_KERNEL_BUNDLE_PROGRAM(data_init);
     ADD_SPLIT_KERNEL_BUNDLE_PROGRAM(state_buffer_size);
@@ -403,7 +394,7 @@ class OpenCLSplitKernel : public DeviceSplitKernel {
         device,
         program_name,
         device->get_opencl_program_filename(kernel_name),
-        device->get_build_options(requested_features, program_name, device->use_preview_kernels));
+        device->get_build_options(requested_features, program_name));
 
     kernel->program.add_kernel(ustring("path_trace_" + kernel_name));
     kernel->program.load();
@@ -569,6 +560,11 @@ class OpenCLSplitKernel : public DeviceSplitKernel {
     size_t num_elements = max_elements_for_max_buffer_size(kg, data, max_buffer_size);
     int2 global_size = make_int2(max(round_down((int)sqrt(num_elements), 64), 64),
                                  (int)sqrt(num_elements));
+
+    if (device->info.description.find("Intel") != string::npos) {
+      global_size = make_int2(min(512, global_size.x), min(512, global_size.y));
+    }
+
     VLOG(1) << "Global size: " << global_size << ".";
     return global_size;
   }
@@ -612,7 +608,6 @@ OpenCLDevice::OpenCLDevice(DeviceInfo &info, Stats &stats, Profiler &profiler, b
     : Device(info, stats, profiler, background),
       load_kernel_num_compiling(0),
       kernel_programs(this),
-      preview_programs(this),
       memory_manager(this),
       texture_info(this, "__texture_info", MEM_GLOBAL)
 {
@@ -622,7 +617,6 @@ OpenCLDevice::OpenCLDevice(DeviceInfo &info, Stats &stats, Profiler &profiler, b
   cqCommandQueue = NULL;
   device_initialized = false;
   textures_need_update = true;
-  use_preview_kernels = !background;
 
   vector<OpenCLPlatformDevice> usable_devices;
   OpenCLInfo::get_usable_devices(&usable_devices);
@@ -678,9 +672,6 @@ OpenCLDevice::OpenCLDevice(DeviceInfo &info, Stats &stats, Profiler &profiler, b
   device_initialized = true;
 
   split_kernel = new OpenCLSplitKernel(this);
-  if (use_preview_kernels) {
-    load_preview_kernels();
-  }
 }
 
 OpenCLDevice::~OpenCLDevice()
@@ -771,7 +762,7 @@ bool OpenCLDevice::load_kernels(const DeviceRequestedFeatures &requested_feature
   load_required_kernels(requested_features);
 
   vector<OpenCLProgram *> programs;
-  kernel_programs.load_kernels(programs, requested_features, false);
+  kernel_programs.load_kernels(programs, requested_features);
 
   if (!requested_features.use_baking && requested_features.use_denoising) {
     denoising_program = OpenCLProgram(
@@ -849,19 +840,6 @@ void OpenCLDevice::load_required_kernels(const DeviceRequestedFeatures &requeste
   }
 }
 
-void OpenCLDevice::load_preview_kernels()
-{
-  DeviceRequestedFeatures no_features;
-  vector<OpenCLProgram *> programs;
-  preview_programs.load_kernels(programs, no_features, true);
-
-  foreach (OpenCLProgram *program, programs) {
-    if (!program->load()) {
-      load_required_kernel_task_pool.push(function_bind(&OpenCLProgram::compile, program));
-    }
-  }
-}
-
 bool OpenCLDevice::wait_for_availability(const DeviceRequestedFeatures &requested_features)
 {
   if (requested_features.use_baking) {
@@ -869,59 +847,18 @@ bool OpenCLDevice::wait_for_availability(const DeviceRequestedFeatures &requeste
     return true;
   }
 
-  if (background) {
-    load_kernel_task_pool.wait_work();
-    use_preview_kernels = false;
-  }
-  else {
-    /* We use a device setting to determine to load preview kernels or not
-     * Better to check on device level than per kernel as mixing preview and
-     * non-preview kernels does not work due to different data types */
-    if (use_preview_kernels) {
-      use_preview_kernels = load_kernel_num_compiling.load() > 0;
-    }
-  }
+  load_kernel_task_pool.wait_work();
   return split_kernel->load_kernels(requested_features);
 }
 
 OpenCLDevice::OpenCLSplitPrograms *OpenCLDevice::get_split_programs()
 {
-  return use_preview_kernels ? &preview_programs : &kernel_programs;
+  return &kernel_programs;
 }
 
 DeviceKernelStatus OpenCLDevice::get_active_kernel_switch_state()
 {
-  /* Do not switch kernels for background renderings
-   * We do foreground rendering but use the preview kernels
-   * Check for the optimized kernels
-   *
-   * This works also the other way around, where we are using
-   * optimized kernels but new ones are being compiled due
-   * to other features that are needed */
-  if (background) {
-    /* The if-statements below would find the same result,
-     * But as the `finished` method uses a mutex we added
-     * this as an early exit */
-    return DEVICE_KERNEL_USING_FEATURE_KERNEL;
-  }
-
-  bool other_kernels_finished = load_kernel_num_compiling.load() == 0;
-  if (use_preview_kernels) {
-    if (other_kernels_finished) {
-      return DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE;
-    }
-    else {
-      return DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL;
-    }
-  }
-  else {
-    if (other_kernels_finished) {
-      return DEVICE_KERNEL_USING_FEATURE_KERNEL;
-    }
-    else {
-      return DEVICE_KERNEL_FEATURE_KERNEL_INVALID;
-    }
-  }
+  return DEVICE_KERNEL_USING_FEATURE_KERNEL;
 }
 
 void OpenCLDevice::mem_alloc(device_memory &mem)

intern/cycles/kernel/kernel_montecarlo.h

@@ -195,31 +195,108 @@ ccl_device float2 regular_polygon_sample(float corners, float rotation, float u,
 
 ccl_device float3 ensure_valid_reflection(float3 Ng, float3 I, float3 N)
 {
-  float3 R;
-  float NI = dot(N, I);
-  float NgR, threshold;
+  float3 R = 2 * dot(N, I) * N - I;
 
-  /* Check if the incident ray is coming from behind normal N. */
-  if (NI > 0) {
-    /* Normal reflection */
-    R = (2 * NI) * N - I;
-    NgR = dot(Ng, R);
+  /* Reflection rays may always be at least as shallow as the incoming ray. */
+  float threshold = min(0.9f * dot(Ng, I), 0.01f);
+  if (dot(Ng, R) >= threshold) {
+    return N;
+  }
 
-    /* Reflection rays may always be at least as shallow as the incoming ray. */
-    threshold = min(0.9f * dot(Ng, I), 0.01f);
-    if (NgR >= threshold) {
-      return N;
+  /* Form coordinate system with Ng as the Z axis and N inside the X-Z-plane.
+   * The X axis is found by normalizing the component of N that's orthogonal to Ng.
+   * The Y axis isn't actually needed.
+   */
+  float NdotNg = dot(N, Ng);
+  float3 X = normalize(N - NdotNg * Ng);
+
+  /* Keep math expressions. */
+  /* clang-format off */
+  /* Calculate N.z and N.x in the local coordinate system.
+   *
+   * The goal of this computation is to find a N' that is rotated towards Ng just enough
+   * to lift R' above the threshold (here called t), therefore dot(R', Ng) = t.
+   *
+   * According to the standard reflection equation,
+   * this means that we want dot(2*dot(N', I)*N' - I, Ng) = t.
+   *
+   * Since the Z axis of our local coordinate system is Ng, dot(x, Ng) is just x.z, so we get
+   * 2*dot(N', I)*N'.z - I.z = t.
+   *
+   * The rotation is simple to express in the coordinate system we formed -
+   * since N lies in the X-Z-plane, we know that N' will also lie in the X-Z-plane,
+   * so N'.y = 0 and therefore dot(N', I) = N'.x*I.x + N'.z*I.z .
+   *
+   * Furthermore, we want N' to be normalized, so N'.x = sqrt(1 - N'.z^2).
+   *
+   * With these simplifications,
+   * we get the final equation 2*(sqrt(1 - N'.z^2)*I.x + N'.z*I.z)*N'.z - I.z = t.
+   *
+   * The only unknown here is N'.z, so we can solve for that.
+   *
+   * The equation has four solutions in general:
+   *
+   * N'.z = +-sqrt(0.5*(+-sqrt(I.x^2*(I.x^2 + I.z^2 - t^2)) + t*I.z + I.x^2 + I.z^2)/(I.x^2 + I.z^2))
+   * We can simplify this expression a bit by grouping terms:
+   *
+   * a = I.x^2 + I.z^2
+   * b = sqrt(I.x^2 * (a - t^2))
+   * c = I.z*t + a
+   * N'.z = +-sqrt(0.5*(+-b + c)/a)
+   *
+   * Two solutions can immediately be discarded because they're negative so N' would lie in the
+   * lower hemisphere.
+   */
+  /* clang-format on */
+
+  float Ix = dot(I, X), Iz = dot(I, Ng);
+  float Ix2 = sqr(Ix), Iz2 = sqr(Iz);
+  float a = Ix2 + Iz2;
+
+  float b = safe_sqrtf(Ix2 * (a - sqr(threshold)));
+  float c = Iz * threshold + a;
+
+  /* Evaluate both solutions.
+   * In many cases one can be immediately discarded (if N'.z would be imaginary or larger than
+   * one), so check for that first. If no option is viable (might happen in extreme cases like N
+   * being in the wrong hemisphere), give up and return Ng. */
+  float fac = 0.5f / a;
+  float N1_z2 = fac * (b + c), N2_z2 = fac * (-b + c);
+  bool valid1 = (N1_z2 > 1e-5f) && (N1_z2 <= (1.0f + 1e-5f));
+  bool valid2 = (N2_z2 > 1e-5f) && (N2_z2 <= (1.0f + 1e-5f));
+
+  float2 N_new;
+  if (valid1 && valid2) {
+    /* If both are possible, do the expensive reflection-based check. */
+    float2 N1 = make_float2(safe_sqrtf(1.0f - N1_z2), safe_sqrtf(N1_z2));
+    float2 N2 = make_float2(safe_sqrtf(1.0f - N2_z2), safe_sqrtf(N2_z2));
+
+    float R1 = 2 * (N1.x * Ix + N1.y * Iz) * N1.y - Iz;
+    float R2 = 2 * (N2.x * Ix + N2.y * Iz) * N2.y - Iz;
+
+    valid1 = (R1 >= 1e-5f);
+    valid2 = (R2 >= 1e-5f);
+    if (valid1 && valid2) {
+      /* If both solutions are valid, return the one with the shallower reflection since it will be
+       * closer to the input (if the original reflection wasn't shallow, we would not be in this
+       * part of the function). */
+      N_new = (R1 < R2) ? N1 : N2;
+    }
+    else {
+      /* If only one reflection is valid (= positive), pick that one. */
+      N_new = (R1 > R2) ? N1 : N2;
     }
   }
+  else if (valid1 || valid2) {
+    /* Only one solution passes the N'.z criterium, so pick that one. */
+    float Nz2 = valid1 ? N1_z2 : N2_z2;
+    N_new = make_float2(safe_sqrtf(1.0f - Nz2), safe_sqrtf(Nz2));
+  }
   else {
-    /* Bad incident */
-    R = -I;
-    NgR = dot(Ng, R);
-    threshold = 0.01f;
+    return Ng;
   }
 
-  R = R + Ng * (threshold - NgR);            /* Lift the reflection above the threshold. */
-  return normalize(I * len(R) + R * len(I)); /* Find a bisector. */
+  return N_new.x * X + N_new.y * Ng;
 }
 
 CCL_NAMESPACE_END

intern/cycles/kernel/kernel_types.h

@@ -99,27 +99,23 @@ CCL_NAMESPACE_BEGIN
 #define __AO__
 #define __PASSES__
 #define __HAIR__
-
-/* Without these we get an AO render, used by OpenCL preview kernel. */
-#ifndef __KERNEL_AO_PREVIEW__
-#  define __SVM__
-#  define __EMISSION__
-#  define __HOLDOUT__
-#  define __MULTI_CLOSURE__
-#  define __TRANSPARENT_SHADOWS__
-#  define __BACKGROUND_MIS__
-#  define __LAMP_MIS__
-#  define __CAMERA_MOTION__
-#  define __OBJECT_MOTION__
-#  define __BAKING__
-#  define __PRINCIPLED__
-#  define __SUBSURFACE__
-#  define __VOLUME__
-#  define __VOLUME_SCATTER__
-#  define __CMJ__
-#  define __SHADOW_RECORD_ALL__
-#  define __BRANCHED_PATH__
-#endif
+#define __SVM__
+#define __EMISSION__
+#define __HOLDOUT__
+#define __MULTI_CLOSURE__
+#define __TRANSPARENT_SHADOWS__
+#define __BACKGROUND_MIS__
+#define __LAMP_MIS__
+#define __CAMERA_MOTION__
+#define __OBJECT_MOTION__
+#define __BAKING__
+#define __PRINCIPLED__
+#define __SUBSURFACE__
+#define __VOLUME__
+#define __VOLUME_SCATTER__
+#define __CMJ__
+#define __SHADOW_RECORD_ALL__
+#define __BRANCHED_PATH__
 
 /* Device specific features */
 #ifdef __KERNEL_CPU__

intern/cycles/kernel/shaders/node_noise_texture.osl

@@ -25,7 +25,7 @@
  * coordinates to act as a seed since the noise functions don't have seed values.
  * A seed value is needed for generating distortion textures and color outputs.
  * The offset's components are in the range [100, 200], not too high to cause
- * bad precision and not to small to be noticeable. We use float seed because
+ * bad precision and not too small to be noticeable. We use float seed because
  * OSL only support float hashes.
  */
 

intern/cycles/kernel/shaders/stdcycles.h

@@ -84,30 +84,67 @@ closure color principled_hair(normal N,
 closure color henyey_greenstein(float g) BUILTIN;
 closure color absorption() BUILTIN;
 
-normal ensure_valid_reflection(normal Ng, normal I, normal N)
+normal ensure_valid_reflection(normal Ng, vector I, normal N)
 {
   /* The implementation here mirrors the one in kernel_montecarlo.h,
    * check there for an explanation of the algorithm. */
-  vector R;
-  float NI = dot(N, I);
-  float NgR, threshold;
 
-  if (NI > 0) {
-    R = (2 * NI) * N - I;
-    NgR = dot(Ng, R);
-    threshold = min(0.9 * dot(Ng, I), 0.01);
-    if (NgR >= threshold) {
-      return N;
+  float sqr(float x)
+  {
+    return x * x;
+  }
+
+  vector R = 2 * dot(N, I) * N - I;
+
+  float threshold = min(0.9 * dot(Ng, I), 0.01);
+  if (dot(Ng, R) >= threshold) {
+    return N;
+  }
+
+  float NdotNg = dot(N, Ng);
+  vector X = normalize(N - NdotNg * Ng);
+
+  float Ix = dot(I, X), Iz = dot(I, Ng);
+  float Ix2 = sqr(Ix), Iz2 = sqr(Iz);
+  float a = Ix2 + Iz2;
+
+  float b = sqrt(Ix2 * (a - sqr(threshold)));
+  float c = Iz * threshold + a;
+
+  float fac = 0.5 / a;
+  float N1_z2 = fac * (b + c), N2_z2 = fac * (-b + c);
+  int valid1 = (N1_z2 > 1e-5) && (N1_z2 <= (1.0 + 1e-5));
+  int valid2 = (N2_z2 > 1e-5) && (N2_z2 <= (1.0 + 1e-5));
+
+  float N_new_x, N_new_z;
+  if (valid1 && valid2) {
+    float N1_x = sqrt(1.0 - N1_z2), N1_z = sqrt(N1_z2);
+    float N2_x = sqrt(1.0 - N2_z2), N2_z = sqrt(N2_z2);
+
+    float R1 = 2 * (N1_x * Ix + N1_z * Iz) * N1_z - Iz;
+    float R2 = 2 * (N2_x * Ix + N2_z * Iz) * N2_z - Iz;
+
+    valid1 = (R1 >= 1e-5);
+    valid2 = (R2 >= 1e-5);
+    if (valid1 && valid2) {
+      N_new_x = (R1 < R2) ? N1_x : N2_x;
+      N_new_z = (R1 < R2) ? N1_z : N2_z;
+    }
+    else {
+      N_new_x = (R1 > R2) ? N1_x : N2_x;
+      N_new_z = (R1 > R2) ? N1_z : N2_z;
     }
   }
+  else if (valid1 || valid2) {
+    float Nz2 = valid1 ? N1_z2 : N2_z2;
+    N_new_x = sqrt(1.0 - Nz2);
+    N_new_z = sqrt(Nz2);
+  }
   else {
-    R = -I;
-    NgR = dot(Ng, R);
-    threshold = 0.01;
+    return Ng;
   }
 
-  R = R + Ng * (threshold - NgR);
-  return normalize(I * length(R) + R * length(I));
+  return N_new_x * X + N_new_z * Ng;
 }
 
 #endif /* CCL_STDOSL_H */

intern/cycles/kernel/svm/svm_noisetex.h

@@ -20,7 +20,7 @@ CCL_NAMESPACE_BEGIN
  * coordinates to act as a seed since the noise functions don't have seed values.
  * A seed value is needed for generating distortion textures and color outputs.
  * The offset's components are in the range [100, 200], not too high to cause
- * bad precision and not to small to be noticeable. We use float seed because
+ * bad precision and not too small to be noticeable. We use float seed because
  * OSL only support float hashes.
  */
 

intern/cycles/render/alembic.cpp

@@ -498,12 +498,8 @@ void AlembicObject::load_data_in_cache(CachedData &cached_data,
 
   /* Use the schema as the base compound property to also be able to look for top level properties.
    */
-  read_attributes(proc,
-                  cached_data,
-                  schema.getArbGeomParams(),
-                  schema.getUVsParam(),
-                  get_requested_attributes(),
-                  progress);
+  read_attributes(
+      proc, cached_data, schema, schema.getUVsParam(), get_requested_attributes(), progress);
 
   cached_data.invalidate_last_loaded_time(true);
   data_loaded = true;
@@ -658,8 +654,7 @@ static void update_attributes(AttributeSet &attributes, CachedData &cached_data,
   list<Attribute>::iterator it;
   for (it = attributes.attributes.begin(); it != attributes.attributes.end();) {
     if (cached_attributes.find(&(*it)) == cached_attributes.end()) {
-      attributes.attributes.erase(it++);
-      attributes.modified = true;
+      attributes.remove(it++);
       continue;
     }
 

intern/cycles/render/alembic_read.cpp

@@ -606,7 +606,8 @@ void read_geometry_data(AlembicProcedural *proc,
 
 template<typename T> struct value_type_converter {
   using cycles_type = float;
-  static constexpr TypeDesc type_desc = TypeFloat;
+  /* Use `TypeDesc::FLOAT` instead of `TypeFloat` to work around a compiler bug in gcc 11. */
+  static constexpr TypeDesc type_desc = TypeDesc::FLOAT;
   static constexpr const char *type_name = "float (default)";
 
   static cycles_type convert_value(T value)
@@ -894,6 +895,10 @@ static void parse_requested_attributes_recursive(const AttributeRequestSet &requ
                                                  const ICompoundProperty &arb_geom_params,
                                                  vector<PropHeaderAndParent> &requested_properties)
 {
+  if (!arb_geom_params.valid()) {
+    return;
+  }
+
   for (const AttributeRequest &req : requested_attributes.requests) {
     const PropertyHeader *property_header = arb_geom_params.getPropertyHeader(req.name.c_str());
 

intern/cycles/render/attribute.cpp

@@ -383,6 +383,23 @@ AttributeStandard Attribute::name_standard(const char *name)
   return ATTR_STD_NONE;
 }
 
+AttrKernelDataType Attribute::kernel_type(const Attribute &attr)
+{
+  if (attr.element == ATTR_ELEMENT_CORNER) {
+    return AttrKernelDataType::UCHAR4;
+  }
+
+  if (attr.type == TypeDesc::TypeFloat) {
+    return AttrKernelDataType::FLOAT;
+  }
+
+  if (attr.type == TypeFloat2) {
+    return AttrKernelDataType::FLOAT2;
+  }
+
+  return AttrKernelDataType::FLOAT3;
+}
+
 void Attribute::get_uv_tiles(Geometry *geom,
                              AttributePrimitive prim,
                              unordered_set<int> &tiles) const
@@ -417,7 +434,7 @@ void Attribute::get_uv_tiles(Geometry *geom,
 /* Attribute Set */
 
 AttributeSet::AttributeSet(Geometry *geometry, AttributePrimitive prim)
-    : geometry(geometry), prim(prim)
+    : modified_flag(~0u), geometry(geometry), prim(prim)
 {
 }
 
@@ -440,7 +457,7 @@ Attribute *AttributeSet::add(ustring name, TypeDesc type, AttributeElement eleme
 
   Attribute new_attr(name, type, element, geometry, prim);
   attributes.emplace_back(std::move(new_attr));
-  modified = true;
+  tag_modified(attributes.back());
   return &attributes.back();
 }
 
@@ -462,8 +479,7 @@ void AttributeSet::remove(ustring name)
 
     for (it = attributes.begin(); it != attributes.end(); it++) {
       if (&*it == attr) {
-        modified = true;
-        attributes.erase(it);
+        remove(it);
         return;
       }
     }
@@ -608,8 +624,7 @@ void AttributeSet::remove(AttributeStandard std)
 
     for (it = attributes.begin(); it != attributes.end(); it++) {
       if (&*it == attr) {
-        modified = true;
-        attributes.erase(it);
+        remove(it);
         return;
       }
     }
@@ -634,6 +649,12 @@ void AttributeSet::remove(Attribute *attribute)
   }
 }
 
+void AttributeSet::remove(list<Attribute>::iterator it)
+{
+  tag_modified(*it);
+  attributes.erase(it);
+}
+
 void AttributeSet::resize(bool reserve_only)
 {
   foreach (Attribute &attr, attributes) {
@@ -674,15 +695,13 @@ void AttributeSet::update(AttributeSet &&new_attributes)
   for (it = attributes.begin(); it != attributes.end();) {
     if (it->std != ATTR_STD_NONE) {
       if (new_attributes.find(it->std) == nullptr) {
-        modified = true;
-        attributes.erase(it++);
+        remove(it++);
         continue;
       }
     }
     else if (it->name != "") {
       if (new_attributes.find(it->name) == nullptr) {
-        modified = true;
-        attributes.erase(it++);
+        remove(it++);
         continue;
       }
     }
@@ -699,7 +718,27 @@ void AttributeSet::clear_modified()
   foreach (Attribute &attr, attributes) {
     attr.modified = false;
   }
-  modified = false;
+
+  modified_flag = 0;
+}
+
+void AttributeSet::tag_modified(const Attribute &attr)
+{
+  /* Some attributes are not stored in the various kernel attribute arrays
+   * (DeviceScene::attribute_*), so the modified flags are only set if the associated standard
+   * corresponds to an attribute which will be stored in the kernel's attribute arrays. */
+  const bool modifies_device_array = (attr.std != ATTR_STD_FACE_NORMAL &&
+                                      attr.std != ATTR_STD_VERTEX_NORMAL);
+
+  if (modifies_device_array) {
+    AttrKernelDataType kernel_type = Attribute::kernel_type(attr);
+    modified_flag |= (1u << kernel_type);
+  }
+}
+
+bool AttributeSet::modified(AttrKernelDataType kernel_type) const
+{
+  return (modified_flag & (1u << kernel_type)) != 0;
 }
 
 /* AttributeRequest */

intern/cycles/render/attribute.h

@@ -39,6 +39,21 @@ class Hair;
 class Mesh;
 struct Transform;
 
+/* AttrKernelDataType.
+ *
+ * The data type of the device arrays storing the attribute's data. Those data types are different
+ * than the ones for attributes as some attribute types are stored in the same array, e.g. Point,
+ * Vector, and Transform are all stored as float3 in the kernel.
+ *
+ * The values of this enumeration are also used as flags to detect changes in AttributeSet. */
+
+enum AttrKernelDataType {
+  FLOAT = 0,
+  FLOAT2 = 1,
+  FLOAT3 = 2,
+  UCHAR4 = 3,
+};
+
 /* Attribute
  *
  * Arbitrary data layers on meshes.
@@ -167,6 +182,8 @@ class Attribute {
   static const char *standard_name(AttributeStandard std);
   static AttributeStandard name_standard(const char *name);
 
+  static AttrKernelDataType kernel_type(const Attribute &attr);
+
   void get_uv_tiles(Geometry *geom, AttributePrimitive prim, unordered_set<int> &tiles) const;
 };
 
@@ -175,11 +192,12 @@ class Attribute {
  * Set of attributes on a mesh. */
 
 class AttributeSet {
+  uint32_t modified_flag;
+
  public:
   Geometry *geometry;
   AttributePrimitive prim;
   list<Attribute> attributes;
-  bool modified = true;
 
   AttributeSet(Geometry *geometry, AttributePrimitive prim);
   AttributeSet(AttributeSet &&) = default;
@@ -197,6 +215,8 @@ class AttributeSet {
 
   void remove(Attribute *attribute);
 
+  void remove(list<Attribute>::iterator it);
+
   void resize(bool reserve_only = false);
   void clear(bool preserve_voxel_data = false);
 
@@ -204,7 +224,18 @@ class AttributeSet {
    * and remove any attribute not found on the new set from this. */
   void update(AttributeSet &&new_attributes);
 
+  /* Return whether the attributes of the given kernel_type are modified, where "modified" means
+   * that some attributes of the given type were added or removed from this AttributeSet. This does
+   * not mean that the data of the remaining attributes in this AttributeSet were also modified. To
+   * check this, use Attribute.modified. */
+  bool modified(AttrKernelDataType kernel_type) const;
+
   void clear_modified();
+
+ private:
+  /* Set the relevant modified flag for the attribute. Only attributes that are stored in device
+   * arrays will be considered for tagging this AttributeSet as modified. */
+  void tag_modified(const Attribute &attr);
 };
 
 /* AttributeRequest

intern/cycles/render/geometry.cpp

@@ -46,6 +46,12 @@ CCL_NAMESPACE_BEGIN
 
 /* Geometry */
 
+PackFlags operator|=(PackFlags &pack_flags, uint32_t value)
+{
+  pack_flags = (PackFlags)((uint32_t)pack_flags | value);
+  return pack_flags;
+}
+
 NODE_ABSTRACT_DEFINE(Geometry)
 {
   NodeType *type = NodeType::add("geometry_base", NULL);
@@ -824,10 +830,13 @@ void GeometryManager::device_update_attributes(Device *device,
   dscene->attributes_float3.alloc(attr_float3_size);
   dscene->attributes_uchar4.alloc(attr_uchar4_size);
 
-  const bool copy_all_data = dscene->attributes_float.need_realloc() ||
-                             dscene->attributes_float2.need_realloc() ||
-                             dscene->attributes_float3.need_realloc() ||
-                             dscene->attributes_uchar4.need_realloc();
+  /* The order of those flags needs to match that of AttrKernelDataType. */
+  const bool attributes_need_realloc[4] = {
+      dscene->attributes_float.need_realloc(),
+      dscene->attributes_float2.need_realloc(),
+      dscene->attributes_float3.need_realloc(),
+      dscene->attributes_uchar4.need_realloc(),
+  };
 
   size_t attr_float_offset = 0;
   size_t attr_float2_offset = 0;
@@ -846,7 +855,7 @@ void GeometryManager::device_update_attributes(Device *device,
 
       if (attr) {
         /* force a copy if we need to reallocate all the data */
-        attr->modified |= copy_all_data;
+        attr->modified |= attributes_need_realloc[Attribute::kernel_type(*attr)];
       }
 
       update_attribute_element_offset(geom,
@@ -869,7 +878,7 @@ void GeometryManager::device_update_attributes(Device *device,
 
         if (subd_attr) {
           /* force a copy if we need to reallocate all the data */
-          subd_attr->modified |= copy_all_data;
+          subd_attr->modified |= attributes_need_realloc[Attribute::kernel_type(*subd_attr)];
         }
 
         update_attribute_element_offset(mesh,
@@ -900,6 +909,10 @@ void GeometryManager::device_update_attributes(Device *device,
     foreach (AttributeRequest &req, attributes.requests) {
       Attribute *attr = values.find(req);
 
+      if (attr) {
+        attr->modified |= attributes_need_realloc[Attribute::kernel_type(*attr)];
+      }
+
       update_attribute_element_offset(object->geometry,
                                       dscene->attributes_float,
                                       attr_float_offset,
@@ -935,10 +948,10 @@ void GeometryManager::device_update_attributes(Device *device,
   /* copy to device */
   progress.set_status("Updating Mesh", "Copying Attributes to device");
 
-  dscene->attributes_float.copy_to_device();
-  dscene->attributes_float2.copy_to_device();
-  dscene->attributes_float3.copy_to_device();
-  dscene->attributes_uchar4.copy_to_device();
+  dscene->attributes_float.copy_to_device_if_modified();
+  dscene->attributes_float2.copy_to_device_if_modified();
+  dscene->attributes_float3.copy_to_device_if_modified();
+  dscene->attributes_uchar4.copy_to_device_if_modified();
 
   if (progress.get_cancel())
     return;
@@ -1236,7 +1249,16 @@ void GeometryManager::device_update_bvh(Device *device,
 
   const bool can_refit = scene->bvh != nullptr &&
                          (bparams.bvh_layout == BVHLayout::BVH_LAYOUT_OPTIX);
-  const bool pack_all = scene->bvh == nullptr;
+
+  PackFlags pack_flags = PackFlags::PACK_NONE;
+
+  if (scene->bvh == nullptr) {
+    pack_flags |= PackFlags::PACK_ALL;
+  }
+
+  if (dscene->prim_visibility.is_modified()) {
+    pack_flags |= PackFlags::PACK_VISIBILITY;
+  }
 
   BVH *bvh = scene->bvh;
   if (!scene->bvh) {
@@ -1274,10 +1296,14 @@ void GeometryManager::device_update_bvh(Device *device,
 
     pack.root_index = -1;
 
-    if (!pack_all) {
+    if (pack_flags != PackFlags::PACK_ALL) {
       /* if we do not need to recreate the BVH, then only the vertices are updated, so we can
        * safely retake the memory */
       dscene->prim_tri_verts.give_data(pack.prim_tri_verts);
+
+      if ((pack_flags & PackFlags::PACK_VISIBILITY) != 0) {
+        dscene->prim_visibility.give_data(pack.prim_visibility);
+      }
     }
     else {
       /* It is not strictly necessary to skip those resizes we if do not have to repack, as the OS
@@ -1306,13 +1332,21 @@ void GeometryManager::device_update_bvh(Device *device,
     // Iterate over scene mesh list instead of objects, since 'optix_prim_offset' was calculated
     // based on that list, which may be ordered differently from the object list.
     foreach (Geometry *geom, scene->geometry) {
-      if (!pack_all && !geom->is_modified()) {
+      /* Make a copy of the pack_flags so the current geometry's flags do not pollute the others'.
+       */
+      PackFlags geom_pack_flags = pack_flags;
+
+      if (geom->is_modified()) {
+        geom_pack_flags |= PackFlags::PACK_VERTICES;
+      }
+
+      if (geom_pack_flags == PACK_NONE) {
         continue;
       }
 
       const pair<int, uint> &info = geometry_to_object_info[geom];
       pool.push(function_bind(
-          &Geometry::pack_primitives, geom, &pack, info.first, info.second, pack_all));
+          &Geometry::pack_primitives, geom, &pack, info.first, info.second, geom_pack_flags));
     }
     pool.wait_work();
   }
@@ -1347,7 +1381,7 @@ void GeometryManager::device_update_bvh(Device *device,
     dscene->prim_type.steal_data(pack.prim_type);
     dscene->prim_type.copy_to_device();
   }
-  if (pack.prim_visibility.size() && (dscene->prim_visibility.need_realloc() || has_bvh2_layout)) {
+  if (pack.prim_visibility.size() && (dscene->prim_visibility.is_modified() || has_bvh2_layout)) {
     dscene->prim_visibility.steal_data(pack.prim_visibility);
     dscene->prim_visibility.copy_to_device();
   }
@@ -1404,24 +1438,46 @@ static void update_device_flags_attribute(uint32_t &device_update_flags,
       continue;
     }
 
-    if (attr.element == ATTR_ELEMENT_CORNER) {
-      device_update_flags |= ATTR_UCHAR4_MODIFIED;
-    }
-    else if (attr.type == TypeDesc::TypeFloat) {
-      device_update_flags |= ATTR_FLOAT_MODIFIED;
-    }
-    else if (attr.type == TypeFloat2) {
-      device_update_flags |= ATTR_FLOAT2_MODIFIED;
-    }
-    else if (attr.type == TypeDesc::TypeMatrix) {
-      device_update_flags |= ATTR_FLOAT3_MODIFIED;
-    }
-    else if (attr.element != ATTR_ELEMENT_VOXEL) {
-      device_update_flags |= ATTR_FLOAT3_MODIFIED;
+    AttrKernelDataType kernel_type = Attribute::kernel_type(attr);
+
+    switch (kernel_type) {
+      case AttrKernelDataType::FLOAT: {
+        device_update_flags |= ATTR_FLOAT_MODIFIED;
+        break;
+      }
+      case AttrKernelDataType::FLOAT2: {
+        device_update_flags |= ATTR_FLOAT2_MODIFIED;
+        break;
+      }
+      case AttrKernelDataType::FLOAT3: {
+        device_update_flags |= ATTR_FLOAT3_MODIFIED;
+        break;
+      }
+      case AttrKernelDataType::UCHAR4: {
+        device_update_flags |= ATTR_UCHAR4_MODIFIED;
+        break;
+      }
     }
   }
 }
 
+static void update_attribute_realloc_flags(uint32_t &device_update_flags,
+                                           const AttributeSet &attributes)
+{
+  if (attributes.modified(AttrKernelDataType::FLOAT)) {
+    device_update_flags |= ATTR_FLOAT_NEEDS_REALLOC;
+  }
+  if (attributes.modified(AttrKernelDataType::FLOAT2)) {
+    device_update_flags |= ATTR_FLOAT2_NEEDS_REALLOC;
+  }
+  if (attributes.modified(AttrKernelDataType::FLOAT3)) {
+    device_update_flags |= ATTR_FLOAT3_NEEDS_REALLOC;
+  }
+  if (attributes.modified(AttrKernelDataType::UCHAR4)) {
+    device_update_flags |= ATTR_UCHAR4_NEEDS_REALLOC;
+  }
+}
+
 void GeometryManager::device_update_preprocess(Device *device, Scene *scene, Progress &progress)
 {
   if (!need_update() && !need_flags_update) {
@@ -1444,16 +1500,11 @@ void GeometryManager::device_update_preprocess(Device *device, Scene *scene, Pro
   foreach (Geometry *geom, scene->geometry) {
     geom->has_volume = false;
 
-    if (geom->attributes.modified) {
-      device_update_flags |= ATTRS_NEED_REALLOC;
-    }
+    update_attribute_realloc_flags(device_update_flags, geom->attributes);
 
     if (geom->is_mesh()) {
       Mesh *mesh = static_cast<Mesh *>(geom);
-
-      if (mesh->subd_attributes.modified) {
-        device_update_flags |= ATTRS_NEED_REALLOC;
-      }
+      update_attribute_realloc_flags(device_update_flags, mesh->subd_attributes);
     }
 
     foreach (Node *node, geom->get_used_shaders()) {
@@ -1595,6 +1646,10 @@ void GeometryManager::device_update_preprocess(Device *device, Scene *scene, Pro
     }
   }
 
+  if ((update_flags & VISIBILITY_MODIFIED) != 0) {
+    dscene->prim_visibility.tag_modified();
+  }
+
   if (device_update_flags & ATTR_FLOAT_NEEDS_REALLOC) {
     dscene->attributes_map.tag_realloc();
     dscene->attributes_float.tag_realloc();
@@ -1921,7 +1976,8 @@ void GeometryManager::device_update(Device *device,
    * Also update the BVH if the transformations change, we cannot rely on tagging the Geometry
    * as modified in this case, as we may accumulate displacement if the vertices do not also
    * change. */
-  bool need_update_scene_bvh = (scene->bvh == nullptr || (update_flags & TRANSFORM_MODIFIED) != 0);
+  bool need_update_scene_bvh = (scene->bvh == nullptr ||
+                                (update_flags & (TRANSFORM_MODIFIED | VISIBILITY_MODIFIED)) != 0);
   {
     scoped_callback_timer timer([scene](double time) {
       if (scene->update_stats) {
@@ -2007,7 +2063,7 @@ void GeometryManager::device_update(Device *device,
      * for meshes with correct bounding boxes.
      *
      * This wouldn't cause wrong results, just true
-     * displacement might be less optimal ot calculate.
+     * displacement might be less optimal to calculate.
      */
     scene->object_manager->need_flags_update = old_need_object_flags_update;
   }

intern/cycles/render/geometry.h

@@ -43,6 +43,24 @@ class Shader;
 class Volume;
 struct PackedBVH;
 
+/* Flags used to determine which geometry data need to be packed. */
+enum PackFlags : uint32_t {
+  PACK_NONE = 0u,
+
+  /* Pack the geometry information (e.g. triangle or curve keys indices). */
+  PACK_GEOMETRY = (1u << 0),
+
+  /* Pack the vertices, for Meshes and Volumes' bounding meshes. */
+  PACK_VERTICES = (1u << 1),
+
+  /* Pack the visibility flags for each triangle or curve. */
+  PACK_VISIBILITY = (1u << 2),
+
+  PACK_ALL = (PACK_GEOMETRY | PACK_VERTICES | PACK_VISIBILITY),
+};
+
+PackFlags operator|=(PackFlags &pack_flags, uint32_t value);
+
 /* Geometry
  *
  * Base class for geometric types like Mesh and Hair. */
@@ -126,7 +144,10 @@ class Geometry : public Node {
                    int n,
                    int total);
 
-  virtual void pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all) = 0;
+  virtual void pack_primitives(PackedBVH *pack,
+                               int object,
+                               uint visibility,
+                               PackFlags pack_flags) = 0;
 
   /* Check whether the geometry should have own BVH built separately. Briefly,
    * own BVH is needed for geometry, if:
@@ -191,6 +212,8 @@ class GeometryManager {
 
     TRANSFORM_MODIFIED = (1 << 10),
 
+    VISIBILITY_MODIFIED = (1 << 11),
+
     /* tag everything in the manager for an update */
     UPDATE_ALL = ~0u,
 

intern/cycles/render/hair.cpp

@@ -494,38 +494,47 @@ void Hair::pack_curves(Scene *scene,
   }
 }
 
-void Hair::pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all)
+void Hair::pack_primitives(PackedBVH *pack, int object, uint visibility, PackFlags pack_flags)
 {
   if (curve_first_key.empty())
     return;
 
-  /* If the BVH does not have to be recreated, we can bail out. */
-  if (!pack_all) {
-    return;
+  /* Separate loop as other arrays are not initialized if their packing is not required. */
+  if ((pack_flags & PACK_VISIBILITY) != 0) {
+    unsigned int *prim_visibility = &pack->prim_visibility[optix_prim_offset];
+
+    size_t index = 0;
+    for (size_t j = 0; j < num_curves(); ++j) {
+      Curve curve = get_curve(j);
+      for (size_t k = 0; k < curve.num_segments(); ++k, ++index) {
+        prim_visibility[index] = visibility;
+      }
+    }
   }
 
-  unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
-  int *prim_type = &pack->prim_type[optix_prim_offset];
-  unsigned int *prim_visibility = &pack->prim_visibility[optix_prim_offset];
-  int *prim_index = &pack->prim_index[optix_prim_offset];
-  int *prim_object = &pack->prim_object[optix_prim_offset];
-  // 'pack->prim_time' is unused by Embree and OptiX
+  if ((pack_flags & PACK_GEOMETRY) != 0) {
+    unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
+    int *prim_type = &pack->prim_type[optix_prim_offset];
+    int *prim_index = &pack->prim_index[optix_prim_offset];
+    int *prim_object = &pack->prim_object[optix_prim_offset];
+    // 'pack->prim_time' is unused by Embree and OptiX
 
-  uint type = has_motion_blur() ?
-                  ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_MOTION_CURVE_RIBBON :
-                                                   PRIMITIVE_MOTION_CURVE_THICK) :
-                  ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_CURVE_RIBBON : PRIMITIVE_CURVE_THICK);
+    uint type = has_motion_blur() ?
+                    ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_MOTION_CURVE_RIBBON :
+                                                     PRIMITIVE_MOTION_CURVE_THICK) :
+                    ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_CURVE_RIBBON :
+                                                     PRIMITIVE_CURVE_THICK);
 
-  size_t index = 0;
-  for (size_t j = 0; j < num_curves(); ++j) {
-    Curve curve = get_curve(j);
-    for (size_t k = 0; k < curve.num_segments(); ++k, ++index) {
-      prim_tri_index[index] = -1;
-      prim_type[index] = PRIMITIVE_PACK_SEGMENT(type, k);
-      prim_visibility[index] = visibility;
-      // Each curve segment points back to its curve index
-      prim_index[index] = j + prim_offset;
-      prim_object[index] = object;
+    size_t index = 0;
+    for (size_t j = 0; j < num_curves(); ++j) {
+      Curve curve = get_curve(j);
+      for (size_t k = 0; k < curve.num_segments(); ++k, ++index) {
+        prim_tri_index[index] = -1;
+        prim_type[index] = PRIMITIVE_PACK_SEGMENT(type, k);
+        // Each curve segment points back to its curve index
+        prim_index[index] = j + prim_offset;
+        prim_object[index] = object;
+      }
     }
   }
 }

intern/cycles/render/hair.h

@@ -146,7 +146,10 @@ class Hair : public Geometry {
   /* BVH */
   void pack_curves(Scene *scene, float4 *curve_key_co, float4 *curve_data, size_t curvekey_offset);
 
-  void pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all) override;
+  void pack_primitives(PackedBVH *pack,
+                       int object,
+                       uint visibility,
+                       PackFlags pack_flags) override;
 };
 
 CCL_NAMESPACE_END

intern/cycles/render/mesh.cpp

@@ -805,7 +805,7 @@ void Mesh::pack_patches(uint *patch_data, uint vert_offset, uint face_offset, ui
   }
 }
 
-void Mesh::pack_primitives(ccl::PackedBVH *pack, int object, uint visibility, bool pack_all)
+void Mesh::pack_primitives(ccl::PackedBVH *pack, int object, uint visibility, PackFlags pack_flags)
 {
   if (triangles.empty())
     return;
@@ -819,28 +819,38 @@ void Mesh::pack_primitives(ccl::PackedBVH *pack, int object, uint visibility, bo
 
   uint type = has_motion_blur() ? PRIMITIVE_MOTION_TRIANGLE : PRIMITIVE_TRIANGLE;
 
-  if (pack_all) {
-    /* Use optix_prim_offset for indexing as those arrays also contain data for Hair geometries. */
-    unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
-    int *prim_type = &pack->prim_type[optix_prim_offset];
+  /* Separate loop as other arrays are not initialized if their packing is not required. */
+  if ((pack_flags & PackFlags::PACK_VISIBILITY) != 0) {
     unsigned int *prim_visibility = &pack->prim_visibility[optix_prim_offset];
-    int *prim_index = &pack->prim_index[optix_prim_offset];
-    int *prim_object = &pack->prim_object[optix_prim_offset];
-
     for (size_t k = 0; k < num_prims; ++k) {
-      prim_tri_index[k] = (prim_offset + k) * 3;
-      prim_type[k] = type;
-      prim_index[k] = prim_offset + k;
-      prim_object[k] = object;
       prim_visibility[k] = visibility;
     }
   }
 
-  for (size_t k = 0; k < num_prims; ++k) {
-    const Mesh::Triangle t = get_triangle(k);
-    prim_tri_verts[k * 3] = float3_to_float4(verts[t.v[0]]);
-    prim_tri_verts[k * 3 + 1] = float3_to_float4(verts[t.v[1]]);
-    prim_tri_verts[k * 3 + 2] = float3_to_float4(verts[t.v[2]]);
+  if ((pack_flags & PackFlags::PACK_GEOMETRY) != 0) {
+    /* Use optix_prim_offset for indexing as those arrays also contain data for Hair geometries. */
+    unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
+    int *prim_type = &pack->prim_type[optix_prim_offset];
+    int *prim_index = &pack->prim_index[optix_prim_offset];
+    int *prim_object = &pack->prim_object[optix_prim_offset];
+
+    for (size_t k = 0; k < num_prims; ++k) {
+      if ((pack_flags & PackFlags::PACK_GEOMETRY) != 0) {
+        prim_tri_index[k] = (prim_offset + k) * 3;
+        prim_type[k] = type;
+        prim_index[k] = prim_offset + k;
+        prim_object[k] = object;
+      }
+    }
+  }
+
+  if ((pack_flags & PackFlags::PACK_VERTICES) != 0) {
+    for (size_t k = 0; k < num_prims; ++k) {
+      const Mesh::Triangle t = get_triangle(k);
+      prim_tri_verts[k * 3] = float3_to_float4(verts[t.v[0]]);
+      prim_tri_verts[k * 3 + 1] = float3_to_float4(verts[t.v[1]]);
+      prim_tri_verts[k * 3 + 2] = float3_to_float4(verts[t.v[2]]);
+    }
   }
 }
 

intern/cycles/render/mesh.h

@@ -232,7 +232,10 @@ class Mesh : public Geometry {
                   size_t tri_offset);
   void pack_patches(uint *patch_data, uint vert_offset, uint face_offset, uint corner_offset);
 
-  void pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all) override;
+  void pack_primitives(PackedBVH *pack,
+                       int object,
+                       uint visibility,
+                       PackFlags pack_flags) override;
 
   void tessellate(DiagSplit *split);
 

intern/cycles/render/object.cpp

@@ -220,6 +220,10 @@ void Object::tag_update(Scene *scene)
       flag |= ObjectManager::TRANSFORM_MODIFIED;
     }
 
+    if (visibility_is_modified()) {
+      flag |= ObjectManager::VISIBILITY_MODIFIED;
+    }
+
     foreach (Node *node, geometry->get_used_shaders()) {
       Shader *shader = static_cast<Shader *>(node);
       if (shader->get_use_mis() && shader->has_surface_emission)
@@ -914,6 +918,10 @@ void ObjectManager::tag_update(Scene *scene, uint32_t flag)
       geometry_flag |= GeometryManager::TRANSFORM_MODIFIED;
     }
 
+    if ((flag & VISIBILITY_MODIFIED) != 0) {
+      geometry_flag |= GeometryManager::VISIBILITY_MODIFIED;
+    }
+
     scene->geometry_manager->tag_update(scene, geometry_flag);
   }
 

intern/cycles/render/object.h

@@ -134,6 +134,7 @@ class ObjectManager {
     OBJECT_MODIFIED = (1 << 5),
     HOLDOUT_MODIFIED = (1 << 6),
     TRANSFORM_MODIFIED = (1 << 7),
+    VISIBILITY_MODIFIED = (1 << 8),
 
     /* tag everything in the manager for an update */
     UPDATE_ALL = ~0u,

intern/cycles/render/scene.cpp

@@ -542,9 +542,6 @@ bool Scene::update(Progress &progress, bool &kernel_switch_needed)
     DeviceKernelStatus kernel_switch_status = device->get_active_kernel_switch_state();
     kernel_switch_needed = kernel_switch_status == DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE ||
                            kernel_switch_status == DEVICE_KERNEL_FEATURE_KERNEL_INVALID;
-    if (kernel_switch_status == DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL) {
-      progress.set_kernel_status("Compiling render kernels");
-    }
     if (new_kernels_needed || kernel_switch_needed) {
       progress.set_kernel_status("Compiling render kernels");
       device->wait_for_availability(loaded_kernel_features);