Cleanup: remove compilation warning in previous commit.

CollectionLineart does not care about the configurations
in master collection.
Other options are not applicaple for master collection as well.
Hence hiding it.

Reviewed by Dalai Felinto (dfelinto)

Differential Revision: https://developer.blender.org/D11702

.clang-format

@@ -2,7 +2,7 @@
 # Configuration of clang-format
 # =============================
 #
-# Tested to work with versions: 6 to 8.
+# Tested to work with versions: 8 to 11.
 
 # This causes parameters on continuations to align to the opening brace.
 #
@@ -263,8 +263,5 @@ ForEachMacros:
   - MAP_SLOT_PROBING_BEGIN
   - VECTOR_SET_SLOT_PROBING_BEGIN
 
-# Use once we bump the minimum version to version 8.
-# # Without this string literals that in-line 'STRINGIFY' behave strangely (a bug?).
-# StatementMacros:
-#   - PyObject_VAR_HEAD
-#   - STRINGIFY
+StatementMacros:
+  - PyObject_VAR_HEAD

.clang-tidy

@@ -35,7 +35,6 @@ Checks:  >
   -modernize-use-auto,
   -modernize-use-trailing-return-type,
   -modernize-avoid-c-arrays,
-  -modernize-use-equals-default,
   -modernize-use-nodiscard,
   -modernize-loop-convert,
   -modernize-pass-by-value,

CMakeLists.txt

@@ -706,7 +706,7 @@ set_and_warn_dependency(WITH_PYTHON WITH_CYCLES        OFF)
 set_and_warn_dependency(WITH_PYTHON WITH_DRACO         OFF)
 
 if(WITH_DRACO AND NOT WITH_PYTHON_INSTALL)
-  message(WARNING "WITH_DRACO requires WITH_PYTHON_INSTALL to be ON, disabling WITH_DRACO for now")
+  message(STATUS "WITH_DRACO requires WITH_PYTHON_INSTALL to be ON, disabling WITH_DRACO for now")
   set(WITH_DRACO OFF)
 endif()
 
@@ -725,7 +725,7 @@ set_and_warn_dependency(WITH_PUGIXML WITH_OPENIMAGEIO  OFF)
 
 if(WITH_BOOST AND NOT (WITH_CYCLES OR WITH_OPENIMAGEIO OR WITH_INTERNATIONAL OR
    WITH_OPENVDB OR WITH_OPENCOLORIO OR WITH_USD OR WITH_ALEMBIC))
-  message(WARNING "No dependencies need 'WITH_BOOST' forcing WITH_BOOST=OFF")
+  message(STATUS "No dependencies need 'WITH_BOOST' forcing WITH_BOOST=OFF")
   set(WITH_BOOST OFF)
 endif()
 
@@ -775,7 +775,7 @@ if(WITH_INSTALL_PORTABLE)
 endif()
 
 if(WITH_GHOST_SDL OR WITH_HEADLESS)
-  message(WARNING "Disabling Ghost Wayland, X11, Input IME, and OpenXR")
+  message(STATUS "Disabling Ghost Wayland, X11, Input IME, and OpenXR")
   set(WITH_GHOST_WAYLAND OFF)
   set(WITH_GHOST_X11     OFF)
   set(WITH_X11_XINPUT    OFF)
@@ -806,7 +806,7 @@ endif()
 if(NOT WITH_CUDA_DYNLOAD)
   find_package(CUDA)
   if(NOT CUDA_FOUND)
-    message(WARNING "CUDA toolkit not found, using dynamic runtime loading of libraries (WITH_CUDA_DYNLOAD) instead")
+    message(STATUS "CUDA toolkit not found, using dynamic runtime loading of libraries (WITH_CUDA_DYNLOAD) instead")
     set(WITH_CUDA_DYNLOAD ON)
   endif()
 endif()
@@ -1243,7 +1243,7 @@ if(WITH_OPENMP)
       find_library_static(OpenMP_LIBRARIES gomp ${CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES})
     endif()
   else()
-    message(WARNING "OpenMP not found, disabling WITH_OPENMP")
+    message(STATUS "OpenMP not found, disabling WITH_OPENMP")
     set(WITH_OPENMP OFF)
   endif()
 
@@ -1319,7 +1319,7 @@ list(APPEND GL_DEFINITIONS -DGLEW_NO_GLU)
 if(WITH_BULLET AND WITH_SYSTEM_BULLET)
   find_package(Bullet)
   if(NOT BULLET_FOUND)
-    message(WARNING "Bullet not found, disabling WITH_BULLET")
+    message(STATUS "Bullet not found, disabling WITH_BULLET")
     set(WITH_BULLET OFF)
   endif()
 else()
@@ -1526,6 +1526,7 @@ if(CMAKE_COMPILER_IS_GNUCC)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_UNDEF -Wundef)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_FORMAT_SIGN -Wformat-signedness)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_RESTRICT -Wrestrict)
+  ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_SUGGEST_OVERRIDE  -Wno-suggest-override)
 
   # gcc 4.2 gives annoying warnings on every file with this
   if(NOT "${CMAKE_C_COMPILER_VERSION}" VERSION_LESS "4.3")
@@ -1589,6 +1590,8 @@ elseif(CMAKE_C_COMPILER_ID MATCHES "Clang")
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_OVERLOADED_VIRTUAL  -Wno-overloaded-virtual)  # we get a lot of these, if its a problem a dev needs to look into it.
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_SIGN_COMPARE        -Wno-sign-compare)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_INVALID_OFFSETOF    -Wno-invalid-offsetof)
+  # Apple Clang (tested on version 12) doesn't support this flag while LLVM Clang 11 does.
+  ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_SUGGEST_OVERRIDE    -Wno-suggest-override)
 
   # gives too many unfixable warnings
   # ADD_CHECK_C_COMPILER_FLAG(C_WARNINGS C_WARN_UNUSED_MACROS      -Wunused-macros)

GNUmakefile

@@ -128,6 +128,9 @@ Utilities
    * source_archive:
      Create a compressed archive of the source code.
 
+   * source_archive_complete:
+     Create a compressed archive of the source code and all the libraries of dependencies.
+
    * update:
      Updates git and all submodules and svn.
 
@@ -477,6 +480,9 @@ check_smatch: .FORCE
 	cd "$(BUILD_DIR)" ; \
 	$(PYTHON) "$(BLENDER_DIR)/build_files/cmake/cmake_static_check_smatch.py"
 
+check_mypy: .FORCE
+	$(PYTHON) "$(BLENDER_DIR)/source/tools/check_source/check_mypy.py"
+
 check_spelling_py: .FORCE
 	cd "$(BUILD_DIR)" ; \
 	PYTHONIOENCODING=utf_8 $(PYTHON) \
@@ -511,6 +517,13 @@ check_descriptions: .FORCE
 source_archive: .FORCE
 	python3 ./build_files/utils/make_source_archive.py
 
+source_archive_complete: .FORCE
+	cmake -S "$(BLENDER_DIR)/build_files/build_environment" -B"$(BUILD_DIR)/source_archive" \
+		-DCMAKE_BUILD_TYPE_INIT:STRING=$(BUILD_TYPE) -DPACKAGE_USE_UPSTREAM_SOURCES=OFF
+# This assumes CMake is still using a default `PACKAGE_DIR` variable:
+	python3 ./build_files/utils/make_source_archive.py --include-packages "$(BUILD_DIR)/source_archive/packages"
+
+
 INKSCAPE_BIN?="inkscape"
 icons: .FORCE
 	BLENDER_BIN=$(BLENDER_BIN) INKSCAPE_BIN=$(INKSCAPE_BIN) \

build_files/build_environment/cmake/download.cmake

@@ -12,7 +12,7 @@ function(download_source dep)
   if(NOT EXISTS ${TARGET_FILE})
     message("Checking source : ${dep} - source not found downloading from ${TARGET_URI}")
     file(DOWNLOAD ${TARGET_URI} ${TARGET_FILE}
-         TIMEOUT 60  # seconds
+         TIMEOUT 1800  # seconds
          EXPECTED_HASH ${TARGET_HASH_TYPE}=${TARGET_HASH}
          TLS_VERIFY ON
          SHOW_PROGRESS

build_files/build_environment/cmake/openimageio.cmake

@@ -68,7 +68,6 @@ set(OPENIMAGEIO_EXTRA_ARGS
   -DBOOST_LIBRARYDIR=${LIBDIR}/boost/lib/
   -DBoost_NO_SYSTEM_PATHS=ON
   -DBoost_NO_BOOST_CMAKE=ON
-  -OIIO_BUILD_CPP11=ON
   -DUSE_LIBSQUISH=OFF
   -DUSE_QT5=OFF
   -DUSE_NUKE=OFF

build_files/build_environment/cmake/options.cmake

@@ -37,14 +37,8 @@ else(BUILD_MODE STREQUAL "Debug")
 endif()
 
 set(DOWNLOAD_DIR "${CMAKE_CURRENT_BINARY_DIR}/downloads" CACHE STRING "Path for downloaded files")
-# look in blenders source folder for packages directory, if that exists
-# it will our package folder, otherwise it will be in the build folder
-if(EXISTS "${CMAKE_SOURCE_DIR}/../../packages")
-  set(PACKAGE_DIR_DEFAULT "${CMAKE_SOURCE_DIR}/../../packages")
-else()
-  set(PACKAGE_DIR_DEFAULT "${CMAKE_CURRENT_BINARY_DIR}/packages")
-endif()
-set(PACKAGE_DIR ${PACKAGE_DIR_DEFAULT} CACHE STRING "Path for downloaded source files")
+# This path must be hard-coded like this, so that the GNUmakefile knows where it is and can pass it to make_source_archive.py:
+set(PACKAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/packages")
 option(PACKAGE_USE_UPSTREAM_SOURCES "Use soures upstream to download the package sources, when OFF the blender mirror will be used" ON)
 
 file(TO_CMAKE_PATH ${DOWNLOAD_DIR} DOWNLOAD_DIR)

build_files/build_environment/patches/osl.diff

@@ -63,3 +63,19 @@ diff -Naur org/CMakeLists.txt external_osl/CMakeLists.txt
 
  set (OSL_NO_DEFAULT_TEXTURESYSTEM OFF CACHE BOOL "Do not use create a raw OIIO::TextureSystem")
  if (OSL_NO_DEFAULT_TEXTURESYSTEM)
+diff --git a/src/liboslexec/llvm_util.cpp b/src/liboslexec/llvm_util.cpp
+index 445f6400..3d468de2 100644
+--- a/src/liboslexec/llvm_util.cpp
++++ b/src/liboslexec/llvm_util.cpp
+@@ -3430,8 +3430,9 @@ LLVM_Util::call_function (llvm::Value *func, cspan<llvm::Value *> args)
+ #endif
+     //llvm_gen_debug_printf (std::string("start ") + std::string(name));
+ #if OSL_LLVM_VERSION >= 110
+-    OSL_DASSERT(llvm::isa<llvm::Function>(func));
+-    llvm::Value *r = builder().CreateCall(llvm::cast<llvm::Function>(func), llvm::ArrayRef<llvm::Value *>(args.data(), args.size()));
++    llvm::Value* r = builder().CreateCall(
++        llvm::cast<llvm::FunctionType>(func->getType()->getPointerElementType()), func,
++        llvm::ArrayRef<llvm::Value*>(args.data(), args.size()));
+ #else
+     llvm::Value *r = builder().CreateCall (func, llvm::ArrayRef<llvm::Value *>(args.data(), args.size()));
+ #endif

build_files/cmake/Modules/FindOSL.cmake

@@ -75,7 +75,7 @@ FIND_PATH(OSL_SHADER_DIR
     /usr/share/OSL/
     /usr/include/OSL/
   PATH_SUFFIXES
-    shaders
+    share/OSL/shaders
 )
 
 # handle the QUIETLY and REQUIRED arguments and set OSL_FOUND to TRUE if

build_files/cmake/cmake_consistency_check.py

@@ -28,6 +28,14 @@ if sys.version_info.major < 3:
           sys.version.partition(" ")[0])
     sys.exit(1)
 
+import os
+from os.path import (
+    dirname,
+    join,
+    normpath,
+    splitext,
+)
+
 from cmake_consistency_check_config import (
     IGNORE_SOURCE,
     IGNORE_SOURCE_MISSING,
@@ -37,32 +45,35 @@ from cmake_consistency_check_config import (
     BUILD_DIR,
 )
 
-
-import os
-from os.path import (
-    dirname,
-    join,
-    normpath,
-    splitext,
+from typing import (
+    Callable,
+    Dict,
+    Generator,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
 )
 
+
 global_h = set()
 global_c = set()
-global_refs = {}
+global_refs: Dict[str, List[Tuple[str, int]]] = {}
 
 # Flatten `IGNORE_SOURCE_MISSING` to avoid nested looping.
-IGNORE_SOURCE_MISSING = [
+IGNORE_SOURCE_MISSING_FLAT = [
     (k, ignore_path) for k, ig_list in IGNORE_SOURCE_MISSING
     for ignore_path in ig_list
 ]
 
 # Ignore cmake file, path pairs.
-global_ignore_source_missing = {}
-for k, v in IGNORE_SOURCE_MISSING:
+global_ignore_source_missing: Dict[str, List[str]] = {}
+for k, v in IGNORE_SOURCE_MISSING_FLAT:
     global_ignore_source_missing.setdefault(k, []).append(v)
+del IGNORE_SOURCE_MISSING_FLAT
 
 
-def replace_line(f, i, text, keep_indent=True):
+def replace_line(f: str, i: int, text: str, keep_indent: bool = True) -> None:
     file_handle = open(f, 'r')
     data = file_handle.readlines()
     file_handle.close()
@@ -77,7 +88,10 @@ def replace_line(f, i, text, keep_indent=True):
     file_handle.close()
 
 
-def source_list(path, filename_check=None):
+def source_list(
+        path: str,
+        filename_check: Optional[Callable[[str], bool]] = None,
+) -> Generator[str, None, None]:
     for dirpath, dirnames, filenames in os.walk(path):
         # skip '.git'
         dirnames[:] = [d for d in dirnames if not d.startswith(".")]
@@ -88,37 +102,37 @@ def source_list(path, filename_check=None):
 
 
 # extension checking
-def is_cmake(filename):
+def is_cmake(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext == ".cmake") or (filename == "CMakeLists.txt")
 
 
-def is_c_header(filename):
+def is_c_header(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext in {".h", ".hpp", ".hxx", ".hh"})
 
 
-def is_c(filename):
+def is_c(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext in {".c", ".cpp", ".cxx", ".m", ".mm", ".rc", ".cc", ".inl"})
 
 
-def is_c_any(filename):
+def is_c_any(filename: str) -> bool:
     return is_c(filename) or is_c_header(filename)
 
 
-def cmake_get_src(f):
+def cmake_get_src(f: str) -> None:
 
     sources_h = []
     sources_c = []
 
     filen = open(f, "r", encoding="utf8")
-    it = iter(filen)
+    it: Optional[Iterator[str]] = iter(filen)
     found = False
     i = 0
     # print(f)
 
-    def is_definition(l, f, i, name):
+    def is_definition(l: str, f: str, i: int, name: str) -> bool:
         if l.startswith("unset("):
             return False
 
@@ -131,6 +145,7 @@ def cmake_get_src(f):
             if l.endswith(")"):
                 raise Exception("strict formatting not kept 'list(APPEND %s...)' on 1 line %s:%d" % (name, f, i))
             return True
+        return False
 
     while it is not None:
         context_name = ""
@@ -269,7 +284,7 @@ def cmake_get_src(f):
     filen.close()
 
 
-def is_ignore_source(f, ignore_used):
+def is_ignore_source(f: str, ignore_used: List[bool]) -> bool:
     for index, ignore_path in enumerate(IGNORE_SOURCE):
         if ignore_path in f:
             ignore_used[index] = True
@@ -277,7 +292,7 @@ def is_ignore_source(f, ignore_used):
     return False
 
 
-def is_ignore_cmake(f, ignore_used):
+def is_ignore_cmake(f: str, ignore_used: List[bool]) -> bool:
     for index, ignore_path in enumerate(IGNORE_CMAKE):
         if ignore_path in f:
             ignore_used[index] = True
@@ -285,7 +300,7 @@ def is_ignore_cmake(f, ignore_used):
     return False
 
 
-def main():
+def main() -> None:
 
     print("Scanning:", SOURCE_DIR)
 
@@ -359,7 +374,7 @@ def main():
                     if "extern" not in f:
                         i = 1
                         try:
-                            for l in open(f, "r", encoding="utf8"):
+                            for _ in open(f, "r", encoding="utf8"):
                                 i += 1
                         except UnicodeDecodeError:
                             print("Non utf8: %s:%d" % (f, i))

build_files/cmake/cmake_static_check_clang_array.py

@@ -25,6 +25,14 @@ import subprocess
 import sys
 import os
 
+from typing import (
+    Any,
+    Callable,
+    List,
+    Tuple,
+)
+
+
 USE_QUIET = (os.environ.get("QUIET", None) is not None)
 
 CHECKER_IGNORE_PREFIX = [
@@ -43,7 +51,7 @@ CHECKER_ARGS = [
 ]
 
 
-def main():
+def main() -> None:
     source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX)
 
     check_commands = []
@@ -52,18 +60,19 @@ def main():
         # ~if "source/blender" not in c:
         # ~    continue
 
-        cmd = ([CHECKER_BIN] +
-               CHECKER_ARGS +
-               [c] +
-               [("-I%s" % i) for i in inc_dirs] +
-               [("-D%s" % d) for d in defs]
-               )
+        cmd = (
+            [CHECKER_BIN] +
+            CHECKER_ARGS +
+            [c] +
+            [("-I%s" % i) for i in inc_dirs] +
+            [("-D%s" % d) for d in defs]
+        )
 
         check_commands.append((c, cmd))
 
     process_functions = []
 
-    def my_process(i, c, cmd):
+    def my_process(i: int, c: str, cmd: str) -> subprocess.Popen[Any]:
         if not USE_QUIET:
             percent = 100.0 * (i / (len(check_commands) - 1))
             percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"

build_files/cmake/cmake_static_check_cppcheck.py

@@ -25,6 +25,12 @@ import subprocess
 import sys
 import os
 
+from typing import (
+    Any,
+    List,
+)
+
+
 USE_QUIET = (os.environ.get("QUIET", None) is not None)
 
 CHECKER_IGNORE_PREFIX = [
@@ -47,25 +53,26 @@ if USE_QUIET:
     CHECKER_ARGS.append("--quiet")
 
 
-def main():
+def main() -> None:
     source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX)
     source_defines = project_source_info.build_defines_as_args()
 
     check_commands = []
     for c, inc_dirs, defs in source_info:
-        cmd = ([CHECKER_BIN] +
-               CHECKER_ARGS +
-               [c] +
-               [("-I%s" % i) for i in inc_dirs] +
-               [("-D%s" % d) for d in defs] +
-               source_defines
-               )
+        cmd = (
+            [CHECKER_BIN] +
+            CHECKER_ARGS +
+            [c] +
+            [("-I%s" % i) for i in inc_dirs] +
+            [("-D%s" % d) for d in defs] +
+            source_defines
+        )
 
         check_commands.append((c, cmd))
 
     process_functions = []
 
-    def my_process(i, c, cmd):
+    def my_process(i: int, c: str, cmd: List[str]) -> subprocess.Popen[Any]:
         if not USE_QUIET:
             percent = 100.0 * (i / len(check_commands))
             percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"

build_files/cmake/config/blender_release.cmake

@@ -56,10 +56,6 @@ set(WITH_TBB                 ON  CACHE BOOL "" FORCE)
 set(WITH_USD                 ON  CACHE BOOL "" FORCE)
 
 set(WITH_MEM_JEMALLOC          ON  CACHE BOOL "" FORCE)
-set(WITH_CYCLES_CUDA_BINARIES  ON  CACHE BOOL "" FORCE)
-set(WITH_CYCLES_CUBIN_COMPILER OFF CACHE BOOL "" FORCE)
-set(CYCLES_CUDA_BINARIES_ARCH sm_30;sm_35;sm_37;sm_50;sm_52;sm_60;sm_61;sm_70;sm_75;sm_86;compute_75 CACHE STRING "" FORCE)
-set(WITH_CYCLES_DEVICE_OPTIX   ON CACHE BOOL "" FORCE)
 
 # platform dependent options
 if(APPLE)
@@ -80,4 +76,8 @@ if(UNIX AND NOT APPLE)
 endif()
 if(NOT APPLE)
   set(WITH_XR_OPENXR              ON  CACHE BOOL "" FORCE)
+
+  set(WITH_CYCLES_DEVICE_OPTIX    ON  CACHE BOOL "" FORCE)
+  set(WITH_CYCLES_CUDA_BINARIES   ON  CACHE BOOL "" FORCE)
+  set(WITH_CYCLES_CUBIN_COMPILER  OFF CACHE BOOL "" FORCE)
 endif()

build_files/cmake/example_scripts/make_quicky.py

@@ -1,119 +0,0 @@
-#!/usr/bin/env python3
-
-# ##### BEGIN GPL LICENSE BLOCK #####
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License
-#  as published by the Free Software Foundation; either version 2
-#  of the License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software Foundation,
-#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# ##### END GPL LICENSE BLOCK #####
-
-# <pep8 compliant>
-
-
-def print_help(targets):
-    print("CMake quicky wrapper, no valid targets given.")
-    print(" * targets can contain a subset of the full target name.")
-    print(" * arguments with a '-' prefix are passed onto make.")
-    print(" * this must run from the cmake build dir")
-    print(" * alias this with a short command for speedy access, in bash:")
-    print("   alias mk='../blender/build_files/cmake/example_scripts/make_quicky.py'")
-    print("")
-    print(" eg: make_quicky.py -j3 extern python")
-    print(" ...will execute")
-    print(" make -j3 extern_binreloc extern_glew bf_python bf_python_ext blender/fast")
-    print("")
-    print("Target List:")
-    for t in targets:
-        print("    %s" % t)
-    print("...exiting")
-
-
-def main():
-    targets = set()
-
-    # collect targets
-    makefile = open("Makefile", "r")
-    for line in makefile:
-        line = line.rstrip()
-        if not line or line[0] in ". \t@$#":
-            continue
-
-        line = line.split("#", 1)[0]
-        if ":" not in line:
-            continue
-
-        line = line.split(":", 1)[0]
-
-        if "/" in line:  # cmake terget options, dont need these
-            continue
-
-        targets.add(line)
-    makefile.close()
-
-    # remove cmake targets
-    bad = set([
-        "help",
-        "clean",
-        "all",
-        "preinstall",
-        "install",
-        "default_target",
-        "edit_cache",
-        "cmake_force",
-        "rebuild_cache",
-        "depend",
-        "cmake_check_build_system",
-    ])
-
-    targets -= set(bad)
-
-    # parse args
-    targets = list(targets)
-    targets.sort()
-
-    import sys
-    if len(sys.argv) == 1:
-        print_help(targets)
-        return
-
-    targets_new = []
-    args = []
-    for arg in sys.argv[1:]:
-        if arg[0] in "/-":
-            args.append(arg)
-        else:
-            found = False
-            for t in targets:
-                if arg in t and t not in targets_new:
-                    targets_new.append(t)
-                    found = True
-
-            if not found:
-                print("Error '%s' not found in...")
-                for t in targets:
-                    print("    %s" % t)
-                print("...aborting.")
-                return
-
-    # execute
-    cmd = ["make"] + args + targets_new + ["blender/fast"]
-    print("cmake building with targets: %s" % " ".join(targets_new))
-    print("executing: %s" % " ".join(cmd))
-
-    import subprocess
-    subprocess.call(cmd)
-
-
-if __name__ == "__main__":
-    main()

build_files/cmake/macros.cmake

@@ -923,6 +923,10 @@ function(get_blender_version)
   math(EXPR _out_version_major "${_out_version} / 100")
   math(EXPR _out_version_minor "${_out_version} % 100")
 
+  # Zero pad the minor version so `_out_version_minor` is always two characters.
+  # This is needed if the minor version is a single digit.
+  string(REGEX REPLACE "^([0-9])$" "0\\1" _out_version_minor "${_out_version_minor}")
+
   # output vars
   set(BLENDER_VERSION "${_out_version_major}.${_out_version_minor}" PARENT_SCOPE)
   set(BLENDER_VERSION_MAJOR "${_out_version_major}" PARENT_SCOPE)

build_files/cmake/platform/platform_apple.cmake

@@ -20,12 +20,6 @@
 
 # Libraries configuration for Apple.
 
-if("${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
-  set(MACOSX_DEPLOYMENT_TARGET 11.00)
-else()
-  set(MACOSX_DEPLOYMENT_TARGET 10.13)
-endif()
-
 macro(find_package_wrapper)
 # do nothing, just satisfy the macro
 endmacro()
@@ -66,7 +60,7 @@ if(WITH_JACK)
     NAMES jackmp
   )
   if(NOT JACK_FRAMEWORK)
-    message(WARNING "JACK not found, disabling WITH_JACK")
+    message(STATUS "JACK not found, disabling WITH_JACK")
     set(WITH_JACK OFF)
   else()
     set(JACK_INCLUDE_DIRS ${JACK_FRAMEWORK}/headers)
@@ -104,6 +98,7 @@ endif()
 if(WITH_USD)
   find_package(USD)
   if(NOT USD_FOUND)
+    message(STATUS "USD not found, disabling WITH_USD")
     set(WITH_USD OFF)
   endif()
 endif()
@@ -310,7 +305,7 @@ if(WITH_OPENCOLORIO)
 
   if(NOT OPENCOLORIO_FOUND)
     set(WITH_OPENCOLORIO OFF)
-    message(STATUS "OpenColorIO not found")
+    message(STATUS "OpenColorIO not found, disabling WITH_OPENCOLORIO")
   endif()
 endif()
 
@@ -387,7 +382,7 @@ if(WITH_OPENIMAGEDENOISE)
 
   if(NOT OPENIMAGEDENOISE_FOUND)
     set(WITH_OPENIMAGEDENOISE OFF)
-    message(STATUS "OpenImageDenoise not found")
+    message(STATUS "OpenImageDenoise not found, disabling WITH_OPENIMAGEDENOISE")
   endif()
 endif()
 

build_files/cmake/platform/platform_apple_xcode.cmake

@@ -168,21 +168,15 @@ endif()
 unset(OSX_SDKROOT)
 
 
-# 10.13 is our min. target, if you use higher sdk, weak linking happens
 if("${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
+  # M1 chips run Big Sur onwards.
   set(OSX_MIN_DEPLOYMENT_TARGET 11.00)
 else()
+  # 10.13 is our min. target, if you use higher sdk, weak linking happens
   set(OSX_MIN_DEPLOYMENT_TARGET 10.13)
 endif()
 
-if(CMAKE_OSX_DEPLOYMENT_TARGET)
-  if(${CMAKE_OSX_DEPLOYMENT_TARGET} VERSION_LESS ${OSX_MIN_DEPLOYMENT_TARGET})
-    message(STATUS "Setting deployment target to ${OSX_MIN_DEPLOYMENT_TARGET}, lower versions are not supported")
-    set(CMAKE_OSX_DEPLOYMENT_TARGET "${OSX_MIN_DEPLOYMENT_TARGET}" CACHE STRING "" FORCE)
-  endif()
-else()
-  set(CMAKE_OSX_DEPLOYMENT_TARGET "${OSX_MIN_DEPLOYMENT_TARGET}" CACHE STRING "" FORCE)
-endif()
+set(CMAKE_OSX_DEPLOYMENT_TARGET "${OSX_MIN_DEPLOYMENT_TARGET}" CACHE STRING "" FORCE)
 
 if(NOT ${CMAKE_GENERATOR} MATCHES "Xcode")
   # Force CMAKE_OSX_DEPLOYMENT_TARGET for makefiles, will not work else (CMake bug?)

build_files/cmake/platform/platform_win32_bundle_crt.cmake

@@ -15,6 +15,15 @@ if(WITH_WINDOWS_BUNDLE_CRT)
 
   include(InstallRequiredSystemLibraries)
 
+  # ucrtbase(d).dll cannot be in the manifest, due to the way windows 10 handles
+  # redirects for this dll, for details see T88813.
+  foreach(lib ${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS})
+    string(FIND ${lib} "ucrtbase" pos)
+    if(NOT pos EQUAL -1)
+      list(REMOVE_ITEM CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS ${lib})
+      install(FILES ${lib} DESTINATION . COMPONENT Libraries)
+    endif()
+  endforeach()
   # Install the CRT to the blender.crt Sub folder.
   install(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS} DESTINATION ./blender.crt COMPONENT Libraries)
 

build_files/cmake/project_info.py

@@ -44,6 +44,15 @@ __all__ = (
     "init",
 )
 
+from typing import (
+    Callable,
+    Generator,
+    List,
+    Optional,
+    Union,
+    Tuple,
+)
+
 
 import sys
 if sys.version_info.major < 3:
@@ -70,10 +79,11 @@ SOURCE_DIR = abspath(SOURCE_DIR)
 SIMPLE_PROJECTFILE = False
 
 # must initialize from 'init'
-CMAKE_DIR = None
+CMAKE_DIR = ""
+PROJECT_DIR = ""
 
 
-def init(cmake_path):
+def init(cmake_path: str) -> bool:
     global CMAKE_DIR, PROJECT_DIR
 
     # get cmake path
@@ -91,7 +101,10 @@ def init(cmake_path):
     return True
 
 
-def source_list(path, filename_check=None):
+def source_list(
+        path: str,
+        filename_check: Optional[Callable[[str], bool]] = None,
+) -> Generator[str, None, None]:
     for dirpath, dirnames, filenames in os.walk(path):
         # skip '.git'
         dirnames[:] = [d for d in dirnames if not d.startswith(".")]
@@ -103,53 +116,57 @@ def source_list(path, filename_check=None):
 
 
 # extension checking
-def is_cmake(filename):
+def is_cmake(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext == ".cmake") or (filename.endswith("CMakeLists.txt"))
 
 
-def is_c_header(filename):
+def is_c_header(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext in {".h", ".hpp", ".hxx", ".hh"})
 
 
-def is_py(filename):
+def is_py(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext == ".py")
 
 
-def is_glsl(filename):
+def is_glsl(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext == ".glsl")
 
 
-def is_c(filename):
+def is_c(filename: str) -> bool:
     ext = splitext(filename)[1]
     return (ext in {".c", ".cpp", ".cxx", ".m", ".mm", ".rc", ".cc", ".inl", ".osl"})
 
 
-def is_c_any(filename):
+def is_c_any(filename: str) -> bool:
     return is_c(filename) or is_c_header(filename)
 
 
-def is_svn_file(filename):
+def is_svn_file(filename: str) -> bool:
     dn, fn = os.path.split(filename)
     filename_svn = join(dn, ".svn", "text-base", "%s.svn-base" % fn)
     return exists(filename_svn)
 
 
-def is_project_file(filename):
+def is_project_file(filename: str) -> bool:
     return (is_c_any(filename) or is_cmake(filename) or is_glsl(filename))  # and is_svn_file(filename)
 
 
-def cmake_advanced_info():
+def cmake_advanced_info() -> Union[Tuple[List[str], List[Tuple[str, str]]], Tuple[None, None]]:
     """ Extract includes and defines from cmake.
     """
 
     make_exe = cmake_cache_var("CMAKE_MAKE_PROGRAM")
+    if make_exe is None:
+        print("Make command not found in: %r not found" % project_path)
+        return None, None
+
     make_exe_basename = os.path.basename(make_exe)
 
-    def create_eclipse_project():
+    def create_eclipse_project() -> str:
         print("CMAKE_DIR %r" % CMAKE_DIR)
         if sys.platform == "win32":
             raise Exception("Error: win32 is not supported")
@@ -219,7 +236,7 @@ def cmake_advanced_info():
     return includes, defines
 
 
-def cmake_cache_var(var):
+def cmake_cache_var(var: str) -> Optional[str]:
     with open(os.path.join(CMAKE_DIR, "CMakeCache.txt"), encoding='utf-8') as cache_file:
         lines = [
             l_strip for l in cache_file
@@ -233,12 +250,12 @@ def cmake_cache_var(var):
     return None
 
 
-def cmake_compiler_defines():
+def cmake_compiler_defines() -> Optional[List[str]]:
     compiler = cmake_cache_var("CMAKE_C_COMPILER")  # could do CXX too
 
     if compiler is None:
         print("Couldn't find the compiler, os defines will be omitted...")
-        return
+        return None
 
     import tempfile
     temp_c = tempfile.mkstemp(suffix=".c")[1]
@@ -255,5 +272,5 @@ def cmake_compiler_defines():
     return lines
 
 
-def project_name_get():
+def project_name_get() -> Optional[str]:
     return cmake_cache_var("CMAKE_PROJECT_NAME")

build_files/cmake/project_source_info.py

@@ -34,30 +34,45 @@ if sys.version_info.major < 3:
 import os
 from os.path import join, dirname, normpath, abspath
 
+import subprocess
+
+from typing import (
+    Any,
+    Callable,
+    Generator,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
+
+
 SOURCE_DIR = join(dirname(__file__), "..", "..")
 SOURCE_DIR = normpath(SOURCE_DIR)
 SOURCE_DIR = abspath(SOURCE_DIR)
 
 
-def is_c_header(filename):
+def is_c_header(filename: str) -> bool:
     ext = os.path.splitext(filename)[1]
     return (ext in {".h", ".hpp", ".hxx", ".hh"})
 
 
-def is_c(filename):
+def is_c(filename: str) -> bool:
     ext = os.path.splitext(filename)[1]
     return (ext in {".c", ".cpp", ".cxx", ".m", ".mm", ".rc", ".cc", ".inl", ".osl"})
 
 
-def is_c_any(filename):
-    return os.path.s_c(filename) or is_c_header(filename)
+def is_c_any(filename: str) -> bool:
+    return is_c(filename) or is_c_header(filename)
 
 
 # copied from project_info.py
 CMAKE_DIR = "."
 
 
-def cmake_cache_var_iter():
+def cmake_cache_var_iter() -> Generator[Tuple[str, str, str], None, None]:
     import re
     re_cache = re.compile(r'([A-Za-z0-9_\-]+)?:?([A-Za-z0-9_\-]+)?=(.*)$')
     with open(join(CMAKE_DIR, "CMakeCache.txt"), 'r', encoding='utf-8') as cache_file:
@@ -68,14 +83,22 @@ def cmake_cache_var_iter():
                 yield (var, type_ or "", val)
 
 
-def cmake_cache_var(var):
+def cmake_cache_var(var: str) -> Optional[str]:
     for var_iter, type_iter, value_iter in cmake_cache_var_iter():
         if var == var_iter:
             return value_iter
     return None
 
 
-def do_ignore(filepath, ignore_prefix_list):
+def cmake_cache_var_or_exit(var: str) -> str:
+    value = cmake_cache_var(var)
+    if value is None:
+        print("Unable to find %r exiting!" % value)
+        sys.exit(1)
+    return value
+
+
+def do_ignore(filepath: str, ignore_prefix_list: Optional[Sequence[str]]) -> bool:
     if ignore_prefix_list is None:
         return False
 
@@ -83,12 +106,13 @@ def do_ignore(filepath, ignore_prefix_list):
     return any([relpath.startswith(prefix) for prefix in ignore_prefix_list])
 
 
-def makefile_log():
+def makefile_log() -> List[str]:
     import subprocess
     import time
 
     # support both make and ninja
-    make_exe = cmake_cache_var("CMAKE_MAKE_PROGRAM")
+    make_exe = cmake_cache_var_or_exit("CMAKE_MAKE_PROGRAM")
+
     make_exe_basename = os.path.basename(make_exe)
 
     if make_exe_basename.startswith(("make", "gmake")):
@@ -102,26 +126,37 @@ def makefile_log():
                                    stdout=subprocess.PIPE,
                                    )
 
+    if process is None:
+        print("Can't execute process")
+        sys.exit(1)
+
+
     while process.poll():
         time.sleep(1)
 
-    out = process.stdout.read()
-    process.stdout.close()
+    # We know this is always true based on the input arguments to `Popen`.
+    stdout: IO[bytes] = process.stdout # type: ignore
+
+    out = stdout.read()
+    stdout.close()
     print("done!", len(out), "bytes")
-    return out.decode("utf-8", errors="ignore").split("\n")
+    return cast(List[str], out.decode("utf-8", errors="ignore").split("\n"))
 
 
-def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None):
-
+def build_info(
+        use_c: bool = True,
+        use_cxx: bool = True,
+        ignore_prefix_list: Optional[List[str]] = None,
+) -> List[Tuple[str, List[str], List[str]]]:
     makelog = makefile_log()
 
     source = []
 
     compilers = []
     if use_c:
-        compilers.append(cmake_cache_var("CMAKE_C_COMPILER"))
+        compilers.append(cmake_cache_var_or_exit("CMAKE_C_COMPILER"))
     if use_cxx:
-        compilers.append(cmake_cache_var("CMAKE_CXX_COMPILER"))
+        compilers.append(cmake_cache_var_or_exit("CMAKE_CXX_COMPILER"))
 
     print("compilers:", " ".join(compilers))
 
@@ -131,7 +166,7 @@ def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None):
 
     for line in makelog:
 
-        args = line.split()
+        args: Union[str, List[str]] = line.split()
 
         if not any([(c in args) for c in compilers]):
             continue
@@ -176,29 +211,40 @@ def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None):
     return source
 
 
-def build_defines_as_source():
+def build_defines_as_source() -> str:
     """
     Returns a string formatted as an include:
         '#defines A=B\n#define....'
     """
     import subprocess
     # works for both gcc and clang
-    cmd = (cmake_cache_var("CMAKE_C_COMPILER"), "-dM", "-E", "-")
-    return subprocess.Popen(cmd,
-                            stdout=subprocess.PIPE,
-                            stdin=subprocess.DEVNULL,
-                            ).stdout.read().strip().decode('ascii')
+    cmd = (cmake_cache_var_or_exit("CMAKE_C_COMPILER"), "-dM", "-E", "-")
+    process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stdin=subprocess.DEVNULL,
+    )
+
+    # We know this is always true based on the input arguments to `Popen`.
+    stdout: IO[bytes] = process.stdout # type: ignore
+
+    return cast(str, stdout.read().strip().decode('ascii'))
 
 
-def build_defines_as_args():
-    return [("-D" + "=".join(l.split(maxsplit=2)[1:]))
-            for l in build_defines_as_source().split("\n")
-            if l.startswith('#define')]
+def build_defines_as_args() -> List[str]:
+    return [
+        ("-D" + "=".join(l.split(maxsplit=2)[1:]))
+        for l in build_defines_as_source().split("\n")
+        if l.startswith('#define')
+    ]
 
 
 # could be moved elsewhere!, this just happens to be used by scripts that also
 # use this module.
-def queue_processes(process_funcs, job_total=-1):
+def queue_processes(
+        process_funcs: Sequence[Tuple[Callable[..., subprocess.Popen[Any]], Tuple[Any, ...]]],
+        job_total: int =-1,
+) -> None:
     """ Takes a list of function arg pairs, each function must return a process
     """
 
@@ -217,7 +263,7 @@ def queue_processes(process_funcs, job_total=-1):
     else:
         import time
 
-        processes = []
+        processes: List[subprocess.Popen[Any]] = []
         for func, args in process_funcs:
             # wait until a thread is free
             while 1:
@@ -234,7 +280,7 @@ def queue_processes(process_funcs, job_total=-1):
             processes.append(func(*args))
 
 
-def main():
+def main() -> None:
     if not os.path.exists(join(CMAKE_DIR, "CMakeCache.txt")):
         print("This script must run from the cmake build dir")
         return

build_files/config/pipeline_config.json

@@ -0,0 +1,91 @@
+{
+    "update-code": {
+        "git": {
+            "submodules": [
+                {
+                    "path": "release/scripts/addons",
+                    "branch": "blender-v2.93-release",
+                    "commit_id": "HEAD"
+                },
+                {
+                    "path": "release/scripts/addons_contrib",
+                    "branch": "blender-v2.93-release",
+                    "commit_id": "HEAD"
+                },
+                {
+                    "path": "release/datafiles/locale",
+                    "branch": "blender-v2.93-release",
+                    "commit_id": "HEAD"
+                },
+                {
+                    "path": "source/tools",
+                    "branch": "blender-v2.93-release",
+                    "commit_id": "HEAD"
+                }
+            ]
+        },
+        "svn": {
+            "tests": {
+                "path": "lib/tests",
+                "branch": "tags/blender-2.93-release",
+                "commit_id": "HEAD"
+            },
+            "libraries": {
+                "darwin-x86_64": {
+                    "path": "lib/darwin",
+                    "branch": "tags/blender-2.93-release",
+                    "commit_id": "HEAD"
+                },
+                "darwin-arm64": {
+                    "path": "lib/darwin_arm64",
+                    "branch": "tags/blender-2.93-release",
+                    "commit_id": "HEAD"
+                },
+                "linux-x86_64": {
+                    "path": "lib/linux_centos7_x86_64",
+                    "branch": "tags/blender-2.93-release",
+                    "commit_id": "HEAD"
+                },
+                "windows-amd64": {
+                    "path": "lib/win64_vc15",
+                    "branch": "tags/blender-2.93-release",
+                    "commit_id": "HEAD"
+                }
+            }
+        }
+    },
+    "buildbot": {
+        "gcc": {
+            "version": "9.0"
+        },
+        "sdks": {
+            "optix": {
+                "version": "7.1.0"
+            },
+            "cuda10": {
+                "version": "10.1"
+            },
+            "cuda11": {
+                "version": "11.3"
+            }
+        },
+        "cmake": {
+            "default": {
+                "version": "any",
+                "overrides": {}
+            },
+            "darwin-x86_64": {
+                "overrides": {}
+            },
+            "darwin-arm64": {
+                "overrides": {}
+            },
+            "linux-x86_64": {
+                "overrides": {}
+            },
+            "windows-amd64": {
+                "overrides": {}
+            }
+        }
+    }
+}

build_files/utils/make_source_archive.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 
+import argparse
 import dataclasses
 import os
 import re
 import subprocess
 from pathlib import Path
-from typing import Iterable, TextIO
+from typing import Iterable, TextIO, Optional, Any, Union
 
 # This script can run from any location,
 # output is created in the $CWD
@@ -18,21 +19,43 @@ SKIP_NAMES = {
     ".gitignore",
     ".gitmodules",
     ".arcconfig",
+    ".svn",
 }
 
 
 def main() -> None:
-    output_dir = Path(".").absolute()
     blender_srcdir = Path(__file__).absolute().parent.parent.parent
+
+    cli_parser = argparse.ArgumentParser(
+        description=f"Create a tarball of the Blender sources, optionally including sources of dependencies.",
+        epilog="This script is intended to be run by `make source_archive_complete`.",
+    )
+    cli_parser.add_argument(
+        "-p",
+        "--include-packages",
+        type=Path,
+        default=None,
+        metavar="PACKAGE_PATH",
+        help="Include all source files from the given package directory as well.",
+    )
+
+    cli_args = cli_parser.parse_args()
+
     print(f"Source dir: {blender_srcdir}")
 
-    version = parse_blender_version(blender_srcdir)
-    manifest = output_dir / f"blender-{version}-manifest.txt"
-    tarball = output_dir / f"blender-{version}.tar.xz"
+    curdir = blender_srcdir.parent
+    os.chdir(curdir)
+    blender_srcdir = blender_srcdir.relative_to(curdir)
 
-    os.chdir(blender_srcdir)
-    create_manifest(version, manifest)
-    create_tarball(version, tarball, manifest)
+    print(f"Output dir: {curdir}")
+
+    version = parse_blender_version(blender_srcdir)
+    tarball = tarball_path(curdir, version, cli_args)
+    manifest = manifest_path(tarball)
+    packages_dir = packages_path(curdir, cli_args)
+
+    create_manifest(version, manifest, blender_srcdir, packages_dir)
+    create_tarball(version, tarball, manifest, blender_srcdir, packages_dir)
     create_checksum_file(tarball)
     cleanup(manifest)
     print("Done!")
@@ -84,43 +107,109 @@ def parse_blender_version(blender_srcdir: Path) -> BlenderVersion:
     )
 
 
+def tarball_path(output_dir: Path, version: BlenderVersion, cli_args: Any) -> Path:
+    extra = ""
+    if cli_args.include_packages:
+        extra = "-with-libraries"
+
+    return output_dir / f"blender{extra}-{version}.tar.xz"
+
+
+def manifest_path(tarball: Path) -> Path:
+    """Return the manifest path for the given tarball path.
+
+    >>> from pathlib import Path
+    >>> tarball = Path("/home/sybren/workspace/blender-git/blender-test.tar.gz")
+    >>> manifest_path(tarball).as_posix()
+    '/home/sybren/workspace/blender-git/blender-test-manifest.txt'
+    """
+    # ".tar.gz" is seen as two suffixes.
+    without_suffix = tarball.with_suffix("").with_suffix("")
+    name = without_suffix.name
+    return without_suffix.with_name(f"{name}-manifest.txt")
+
+
+def packages_path(current_directory: Path, cli_args: Any) -> Optional[Path]:
+    if not cli_args.include_packages:
+        return None
+
+    abspath = cli_args.include_packages.absolute()
+
+    # os.path.relpath() can return paths like "../../packages", where
+    # Path.relative_to() will not go up directories (so its return value never
+    # has "../" in there).
+    relpath = os.path.relpath(abspath, current_directory)
+
+    return Path(relpath)
+
+
 ### Manifest creation
 
 
-def create_manifest(version: BlenderVersion, outpath: Path) -> None:
+def create_manifest(
+    version: BlenderVersion,
+    outpath: Path,
+    blender_srcdir: Path,
+    packages_dir: Optional[Path],
+) -> None:
     print(f'Building manifest of files:  "{outpath}"...', end="", flush=True)
     with outpath.open("w", encoding="utf-8") as outfile:
-        main_files_to_manifest(outfile)
-        submodules_to_manifest(version, outfile)
+        main_files_to_manifest(blender_srcdir, outfile)
+        submodules_to_manifest(blender_srcdir, version, outfile)
+
+        if packages_dir:
+            packages_to_manifest(outfile, packages_dir)
     print("OK")
 
 
-def main_files_to_manifest(outfile: TextIO) -> None:
-    for path in git_ls_files():
+def main_files_to_manifest(blender_srcdir: Path, outfile: TextIO) -> None:
+    assert not blender_srcdir.is_absolute()
+    for path in git_ls_files(blender_srcdir):
         print(path, file=outfile)
 
 
-def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None:
+def submodules_to_manifest(
+    blender_srcdir: Path, version: BlenderVersion, outfile: TextIO
+) -> None:
     skip_addon_contrib = version.is_release
+    assert not blender_srcdir.is_absolute()
 
-    for line in git_command("submodule"):
+    for line in git_command("-C", blender_srcdir, "submodule"):
         submodule = line.split()[1]
 
         # Don't use native slashes as GIT for MS-Windows outputs forward slashes.
         if skip_addon_contrib and submodule == "release/scripts/addons_contrib":
             continue
 
-        for path in git_ls_files(Path(submodule)):
+        for path in git_ls_files(blender_srcdir / submodule):
             print(path, file=outfile)
 
 
-def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None:
+def packages_to_manifest(outfile: TextIO, packages_dir: Path) -> None:
+    for path in packages_dir.glob("*"):
+        if not path.is_file():
+            continue
+        if path.name in SKIP_NAMES:
+            continue
+        print(path, file=outfile)
+
+
+### Higher-level functions
+
+
+def create_tarball(
+    version: BlenderVersion, tarball: Path, manifest: Path, blender_srcdir: Path, packages_dir: Optional[Path]
+) -> None:
     print(f'Creating archive:            "{tarball}" ...', end="", flush=True)
+    command = ["tar"]
+
     # Requires GNU `tar`, since `--transform` is used.
-    command = [
-        "tar",
+    if packages_dir:
+        command += ["--transform", f"s,{packages_dir}/,packages/,g"]
+
+    command += [
         "--transform",
-        f"s,^,blender-{version}/,g",
+        f"s,^{blender_srcdir.name}/,blender-{version}/,g",
         "--use-compress-program=xz -9",
         "--create",
         f"--file={tarball}",
@@ -130,7 +219,8 @@ def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> No
         "--owner=0",
         "--group=0",
     ]
-    subprocess.run(command, check=True, timeout=300)
+
+    subprocess.run(command, check=True, timeout=3600)
     print("OK")
 
 
@@ -174,7 +264,7 @@ def git_ls_files(directory: Path = Path(".")) -> Iterable[Path]:
         yield path
 
 
-def git_command(*cli_args) -> Iterable[str]:
+def git_command(*cli_args: Union[bytes, str, Path] ) -> Iterable[str]:
     """Generator, yields lines of output from a Git command."""
     command = ("git", *cli_args)
 

doc/manpage/blender.1.py

@@ -22,7 +22,7 @@
 This script generates the blender.1 man page, embedding the help text
 from the Blender executable itself. Invoke it as follows:
 
-    blender.1.py <path-to-blender> <output-filename>
+    blender.1.py --blender <path-to-blender> --output <output-filename>
 
 where <path-to-blender> is the path to the Blender executable,
 and <output-filename> is where to write the generated man page.
@@ -30,108 +30,147 @@ and <output-filename> is where to write the generated man page.
 
 # <pep8 compliant>
 
+import argparse
 import os
 import subprocess
 import sys
-
 import time
 
+from typing import (
+    Dict,
+    TextIO,
+)
 
-def man_format(data):
+
+def man_format(data: str) -> str:
     data = data.replace("-", "\\-")
     data = data.replace("\t", "  ")
     return data
 
 
-if len(sys.argv) != 3:
-    import getopt
-    raise getopt.GetoptError("Usage: %s <path-to-blender> <output-filename>" % sys.argv[0])
+def blender_extract_info(blender_bin: str) -> Dict[str, str]:
 
-blender_bin = sys.argv[1]
-outfilename = sys.argv[2]
+    blender_env = {
+        "ASAN_OPTIONS": "exitcode=0:" + os.environ.get("ASAN_OPTIONS", ""),
+    }
 
-cmd = [blender_bin, "--help"]
-print("  executing:", " ".join(cmd))
-ASAN_OPTIONS = "exitcode=0:" + os.environ.get("ASAN_OPTIONS", "")
-blender_help = subprocess.run(
-    cmd, env={"ASAN_OPTIONS": ASAN_OPTIONS}, check=True, stdout=subprocess.PIPE).stdout.decode(encoding="utf-8")
-blender_version = subprocess.run(
-    [blender_bin, "--version"], env={"ASAN_OPTIONS": ASAN_OPTIONS}, check=True, stdout=subprocess.PIPE).stdout.decode(encoding="utf-8").strip()
-blender_version, blender_date = (blender_version.split("build") + [None, None])[0:2]
-blender_version = blender_version.rstrip().partition(" ")[2]  # remove 'Blender' prefix.
-if blender_date is None:
-    # Happens when built without WITH_BUILD_INFO e.g.
-    date_string = time.strftime("%B %d, %Y", time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))))
-else:
-    blender_date = blender_date.strip().partition(" ")[2]  # remove 'date:' prefix
-    date_string = time.strftime("%B %d, %Y", time.strptime(blender_date, "%Y-%m-%d"))
+    blender_help = subprocess.run(
+        [blender_bin, "--help"],
+        env=blender_env,
+        check=True,
+        stdout=subprocess.PIPE,
+    ).stdout.decode(encoding="utf-8")
 
-outfile = open(outfilename, "w")
-fw = outfile.write
+    blender_version_ouput = subprocess.run(
+        [blender_bin, "--version"],
+        env=blender_env,
+        check=True,
+        stdout=subprocess.PIPE,
+    ).stdout.decode(encoding="utf-8")
 
-fw('.TH "BLENDER" "1" "%s" "Blender %s"\n' % (date_string, blender_version.replace(".", "\\&.")))
+    # Extract information from the version string.
+    # Note that some internal modules may print errors (e.g. color management),
+    # check for each lines prefix to ensure these aren't included.
+    blender_version = ""
+    blender_date = ""
+    for l in blender_version_ouput.split("\n"):
+        if l.startswith("Blender "):
+            # Remove 'Blender' prefix.
+            blender_version = l.split(" ", 1)[1].strip()
+        elif l.lstrip().startswith("build date:"):
+            # Remove 'build date:' prefix.
+            blender_date = l.split(":", 1)[1].strip()
+        if blender_version and blender_date:
+            break
 
-fw('''
+    if not blender_date:
+        # Happens when built without WITH_BUILD_INFO e.g.
+        date_string = time.strftime("%B %d, %Y", time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))))
+    else:
+        date_string = time.strftime("%B %d, %Y", time.strptime(blender_date, "%Y-%m-%d"))
+
+    return {
+        "help": blender_help,
+        "version": blender_version,
+        "date": date_string,
+    }
+
+
+def man_page_from_blender_help(fh: TextIO, blender_bin: str) -> None:
+    blender_info = blender_extract_info(blender_bin)
+
+    # Header Content.
+    fh.write(
+        '.TH "BLENDER" "1" "%s" "Blender %s"\n' %
+        (blender_info["date"], blender_info["version"].replace(".", "\\&."))
+    )
+
+    fh.write(r'''
 .SH NAME
 blender \- a full-featured 3D application''')
 
-fw('''
+    fh.write(r'''
 .SH SYNOPSIS
 .B blender [args ...] [file] [args ...]''')
 
-fw('''
+    fh.write(r'''
 .br
 .SH DESCRIPTION
 .PP
 .B blender
-is a full-featured 3D application. It supports the entirety of the 3D pipeline - modeling, rigging, animation, simulation, rendering, compositing, motion tracking, and video editing.
+is a full-featured 3D application. It supports the entirety of the 3D pipeline - '''
+'''modeling, rigging, animation, simulation, rendering, compositing, motion tracking, and video editing.
 
-Use Blender to create 3D images and animations, films and commercials, content for games, architectural and industrial visualizatons, and scientific visualizations.
+Use Blender to create 3D images and animations, films and commercials, content for games, '''
+r'''architectural and industrial visualizatons, and scientific visualizations.
 
 https://www.blender.org''')
 
-fw('''
+    fh.write(r'''
 .SH OPTIONS''')
 
-fw("\n\n")
+    fh.write("\n\n")
 
-lines = [line.rstrip() for line in blender_help.split("\n")]
+    # Body Content.
 
-while lines:
-    l = lines.pop(0)
-    if l.startswith("Environment Variables:"):
-        fw('.SH "ENVIRONMENT VARIABLES"\n')
-    elif l.endswith(":"):  # one line
-        fw('.SS "%s"\n\n' % l)
-    elif l.startswith("-") or l.startswith("/"):  # can be multi line
+    lines = [line.rstrip() for line in blender_info["help"].split("\n")]
 
-        fw('.TP\n')
-        fw('.B %s\n' % man_format(l))
+    while lines:
+        l = lines.pop(0)
+        if l.startswith("Environment Variables:"):
+            fh.write('.SH "ENVIRONMENT VARIABLES"\n')
+        elif l.endswith(":"):  # One line.
+            fh.write('.SS "%s"\n\n' % l)
+        elif l.startswith("-") or l.startswith("/"):  # Can be multi line.
+            fh.write('.TP\n')
+            fh.write('.B %s\n' % man_format(l))
 
-        while lines:
-            # line with no
-            if lines[0].strip() and len(lines[0].lstrip()) == len(lines[0]):  # no white space
-                break
+            while lines:
+                # line with no
+                if lines[0].strip() and len(lines[0].lstrip()) == len(lines[0]):  # No white space.
+                    break
 
-            if not l:  # second blank line
-                fw('.IP\n')
-            else:
-                fw('.br\n')
+                if not l:  # Second blank line.
+                    fh.write('.IP\n')
+                else:
+                    fh.write('.br\n')
 
-            l = lines.pop(0)
-            l = l[1:]  # remove first whitespace (tab)
+                l = lines.pop(0)
+                if l:
+                    assert(l.startswith('\t'))
+                    l = l[1:]  # Remove first white-space (tab).
 
-            fw('%s\n' % man_format(l))
+                fh.write('%s\n' % man_format(l))
 
-    else:
-        if not l.strip():
-            fw('.br\n')
         else:
-            fw('%s\n' % man_format(l))
+            if not l.strip():
+                fh.write('.br\n')
+            else:
+                fh.write('%s\n' % man_format(l))
 
-# footer
+    # Footer Content.
 
-fw('''
+    fh.write(r'''
 .br
 .SH SEE ALSO
 .B luxrender(1)
@@ -143,5 +182,33 @@ This manpage was written for a Debian GNU/Linux system by Daniel Mester
 <cyril.brulebois@enst-bretagne.fr> and Dan Eicher <dan@trollwerks.org>.
 ''')
 
-outfile.close()
-print("written:", outfilename)
+
+def create_argparse() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="The man page to write to."
+    )
+    parser.add_argument(
+        "--blender",
+        required=True,
+        help="Path to the blender binary."
+    )
+
+    return parser
+
+
+def main() -> None:
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    blender_bin = args.blender
+    output_filename = args.output
+
+    with open(output_filename, "w", encoding="utf-8") as fh:
+        man_page_from_blender_help(fh, blender_bin)
+
+
+if __name__ == "__main__":
+    main()

doc/python_api/requirements.txt

@@ -1,2 +1,2 @@
-Sphinx==3.5.1
-sphinx_rtd_theme==0.5.1
+Sphinx==3.5.3
+sphinx_rtd_theme==0.5.2

doc/python_api/sphinx_doc_gen.py

@@ -545,6 +545,13 @@ def range_str(val):
 
 
 def example_extract_docstring(filepath):
+    '''
+    Return (text, line_no, line_no_has_content) where:
+    - ``text`` is the doc-string text.
+    - ``line_no`` is the line the doc-string text ends.
+    - ``line_no_has_content`` when False, this file only contains a doc-string.
+      There is no need to include the remainder.
+    '''
     file = open(filepath, "r", encoding="utf-8")
     line = file.readline()
     line_no = 0
@@ -553,7 +560,7 @@ def example_extract_docstring(filepath):
         line_no += 1
     else:
         file.close()
-        return "", 0, False
+        return "", 0, True
 
     for line in file:
         line_no += 1

doc/python_api/static/css/theme_overrides.css

@@ -1,7 +1,8 @@
 /* T76453: Prevent Long enum lists */
-.field-list li {
+.field-list > dd p {
    max-height: 245px;
    overflow-y: auto !important;
+   word-break: break-word;
 }
 
 /* Hide home icon in search area */
@@ -11,3 +12,15 @@
 .wy-nav-content {
     max-width: 1000px !important;
 }
+
+/* Fix long titles on mobile */
+h1, h2, h3, h4, h5, h6 {word-break: break-all}
+
+/* Temp fix for https://github.com/readthedocs/sphinx_rtd_theme/pull/1109 */
+.hlist tr {
+   display: -ms-flexbox;
+   display: flex;
+   flex-flow: row wrap;
+ }
+
+.hlist td {margin-right: auto}

extern/mantaflow/preprocessed/fileio/iovdb.cpp

@@ -29,10 +29,10 @@
 
 #if OPENVDB == 1
 #  include "openvdb/openvdb.h"
-#  include <openvdb/points/PointConversion.h>
-#  include <openvdb/points/PointCount.h>
-#  include <openvdb/tools/Clip.h>
-#  include <openvdb/tools/Dense.h>
+#  include "openvdb/points/PointConversion.h"
+#  include "openvdb/points/PointCount.h"
+#  include "openvdb/tools/Clip.h"
+#  include "openvdb/tools/Dense.h"
 #endif
 
 #define POSITION_NAME "P"
@@ -519,7 +519,7 @@ int writeObjectsVDB(const string &filename,
     }
   }
 
-  // Write only if the is at least one grid, optionally write with compression.
+  // Write only if there is at least one grid, optionally write with compression.
   if (gridsVDB.size()) {
     int vdb_flags = openvdb::io::COMPRESS_ACTIVE_MASK;
     switch (compression) {
@@ -534,7 +534,8 @@ int writeObjectsVDB(const string &filename,
       }
       case COMPRESSION_BLOSC: {
 #  if OPENVDB_BLOSC == 1
-        vdb_flags |= openvdb::io::COMPRESS_BLOSC;
+        // Cannot use |= here, causes segfault with blosc 1.5.0 (== recommended version)
+        vdb_flags = openvdb::io::COMPRESS_BLOSC;
 #  else
         debMsg("OpenVDB was built without Blosc support, using Zip compression instead", 1);
         vdb_flags |= openvdb::io::COMPRESS_ZIP;

extern/mantaflow/preprocessed/fluidsolver.h

@@ -384,6 +384,7 @@ class FluidSolver : public PbClass {
   GridStorage<Real> mGrids4dReal;
   GridStorage<Vec3> mGrids4dVec;
   GridStorage<Vec4> mGrids4dVec4;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/general.h

@@ -42,7 +42,7 @@ inline void updateQtGui(bool full, int frame, float time, const std::string &cur
 #  ifdef _DEBUG
 #    define DEBUG 1
 #  endif  // _DEBUG
-#endif  // DEBUG
+#endif    // DEBUG
 
 // Standard exception
 class Error : public std::exception {

extern/mantaflow/preprocessed/gitinfo.h

@@ -1,3 +1,3 @@
 
 
-#define MANTA_GIT_VERSION "commit 39b7a415721ecbf6643612a24e8eadd221aeb934"
+#define MANTA_GIT_VERSION "commit 9c505cd22e289b98c9aa717efba8ef3201c7e458"

extern/mantaflow/preprocessed/grid.h

@@ -389,6 +389,7 @@ class GridBase : public PbClass {
   Real mDx;
   bool m3D;  // precomputed Z shift: to ensure 2D compatibility, always use this instead of sx*sy !
   IndexInt mStrideZ;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/grid4d.h

@@ -326,6 +326,7 @@ class Grid4dBase : public PbClass {
   // precomputed Z,T shift: to ensure 2D compatibility, always use this instead of sx*sy !
   IndexInt mStrideZ;
   IndexInt mStrideT;
+
  public:
   PbArgs _args;
 }
@@ -950,6 +951,7 @@ template<class T> class Grid4d : public Grid4dBase {
 
  protected:
   T *mData;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/levelset.h

@@ -266,6 +266,7 @@ class LevelsetGrid : public Grid<Real> {
   }
 
   static Real invalidTimeValue();
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/mesh.h

@@ -796,6 +796,7 @@ class Mesh : public PbClass {
   std::vector<MeshDataImpl<int> *>
       mMdataInt;  //! indicate that mdata of this mesh is copied, and needs to be freed
   bool mFreeMdata;
+
  public:
   PbArgs _args;
 }
@@ -881,6 +882,7 @@ class MeshDataBase : public PbClass {
 
  protected:
   Mesh *mMesh;
+
  public:
   PbArgs _args;
 }
@@ -1645,6 +1647,7 @@ template<class T> class MeshDataImpl : public MeshDataBase {
   //! optionally , we might have an associated grid from which to grab new data
   Grid<T> *mpGridSource;  //! unfortunately , we need to distinguish mac vs regular vec3
   bool mGridSourceMAC;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/movingobs.h

@@ -154,6 +154,7 @@ class MovingObstacle : public PbClass {
   int mEmptyType;
   int mID;
   static int sIDcnt;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/noisefield.h

@@ -236,6 +236,7 @@ class WaveletNoiseField : public PbClass {
   static int randomSeed;
   // global reference count for noise tile
   static std::atomic<int> mNoiseReferenceCount;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/particle.h

@@ -205,6 +205,7 @@ class ParticleBase : public PbClass {
   //! custom seed for particle systems, used by plugins
   int mSeed;  //! fix global random seed storage, used mainly by functions in this class
   static int globalSeed;
+
  public:
   PbArgs _args;
 }
@@ -628,6 +629,7 @@ template<class S> class ParticleSystem : public ParticleBase {
   std::vector<S> mData;
   //! reduce storage , called by doCompress
   virtual void compress();
+
  public:
   PbArgs _args;
 }
@@ -918,6 +920,7 @@ class ParticleIndexSystem : public ParticleSystem<ParticleIndexData> {
       return -1;
     }
   };
+
  public:
   PbArgs _args;
 }
@@ -982,6 +985,7 @@ template<class DATA, class CON> class ConnectedParticleSystem : public ParticleS
  protected:
   std::vector<CON> mSegments;
   virtual void compress();
+
  public:
   PbArgs _args;
 }
@@ -1071,6 +1075,7 @@ class ParticleDataBase : public PbClass {
 
  protected:
   ParticleBase *mpParticleSys;
+
  public:
   PbArgs _args;
 }
@@ -1843,6 +1848,7 @@ template<class T> class ParticleDataImpl : public ParticleDataBase {
   //! optionally , we might have an associated grid from which to grab new data
   Grid<T> *mpGridSource;  //! unfortunately , we need to distinguish mac vs regular vec3
   bool mGridSourceMAC;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/plugin/meshplugins.cpp

@@ -234,10 +234,10 @@ void subdivideMesh(
       normalize(ne2);
 
       // Real thisArea = sqrMag(cross(-e2,e0));
-      // small angle approximation says sin(x) = arcsin(x) = x,
-      // arccos(x) = pi/2 - arcsin(x),
-      // cos(x) = dot(A,B),
-      // so angle is approximately 1 - dot(A,B).
+      //  small angle approximation says sin(x) = arcsin(x) = x,
+      //  arccos(x) = pi/2 - arcsin(x),
+      //  cos(x) = dot(A,B),
+      //  so angle is approximately 1 - dot(A,B).
       Real angle[3];
       angle[0] = 1.0 - dot(ne0, -ne2);
       angle[1] = 1.0 - dot(ne1, -ne0);

extern/mantaflow/preprocessed/plugin/secondaryparticles.cpp

@@ -2287,9 +2287,10 @@ struct knFlipComputePotentialTrappedAir : public KernelBase {
           const Vec3 &vj = scaleFromManta * v.getCentered(x, y, z);
           const Vec3 xij = xi - xj;
           const Vec3 vij = vi - vj;
-          Real h = !pot.is3D() ? 1.414 * radius :
-                                 1.732 * radius;  // estimate sqrt(2)*radius resp. sqrt(3)*radius
-                                                  // for h, due to squared resp. cubic neighbor area
+          Real h = !pot.is3D() ?
+                       1.414 * radius :
+                       1.732 * radius;  // estimate sqrt(2)*radius resp. sqrt(3)*radius for h, due
+                                        // to squared resp. cubic neighbor area
           vdiff += norm(vij) * (1 - dot(getNormalized(vij), getNormalized(xij))) *
                    (1 - norm(xij) / h);
         }

extern/mantaflow/preprocessed/shapes.h

@@ -269,6 +269,7 @@ class Shape : public PbClass {
 
  protected:
   GridType mType;
+
  public:
   PbArgs _args;
 }
@@ -319,6 +320,7 @@ class NullShape : public Shape {
   {
     gridSetConst<Real>(phi, 1000.0f);
   }
+
  public:
   PbArgs _args;
 }
@@ -394,6 +396,7 @@ class Box : public Shape {
 
  protected:
   Vec3 mP0, mP1;
+
  public:
   PbArgs _args;
 }
@@ -455,6 +458,7 @@ class Sphere : public Shape {
  protected:
   Vec3 mCenter, mScale;
   Real mRadius;
+
  public:
   PbArgs _args;
 }
@@ -579,6 +583,7 @@ class Cylinder : public Shape {
  protected:
   Vec3 mCenter, mZDir;
   Real mRadius, mZ;
+
  public:
   PbArgs _args;
 }
@@ -655,6 +660,7 @@ class Slope : public Shape {
   Real mAnglexy, mAngleyz;
   Real mOrigin;
   Vec3 mGs;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/turbulencepart.h

@@ -199,6 +199,7 @@ class TurbulenceParticleSystem : public ParticleSystem<TurbulenceParticleData> {
 
  private:
   WaveletNoiseField &noise;
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/vortexpart.h

@@ -127,6 +127,7 @@ class VortexParticleSystem : public ParticleSystem<VortexParticleData> {
   }
 
   virtual ParticleBase *clone();
+
  public:
   PbArgs _args;
 }

extern/mantaflow/preprocessed/vortexsheet.h

@@ -240,6 +240,7 @@ class VortexSheetMesh : public Mesh {
   VorticityChannel mVorticity;
   TexCoord3Channel mTex1, mTex2;
   TurbulenceChannel mTurb;
+
  public:
   PbArgs _args;
 }

extern/smaa_areatex/smaa_areatex.cpp

@@ -1088,7 +1088,7 @@ static int generate_file(AreaOrtho *ortho, AreaDiag *diag, const char *path, boo
 		return 1;
 	}
 
-	fprintf(stderr, "Generating %s\n", path);
+	// fprintf(stderr, "Generating %s\n", path);
 
 	if (tga)
 		write_tga(ortho, diag, fp, subsampling);

intern/clog/CLG_log.h

@@ -118,6 +118,7 @@ typedef struct CLG_LogType {
 typedef struct CLG_LogRef {
   const char *identifier;
   CLG_LogType *type;
+  struct CLG_LogRef *next;
 } CLG_LogRef;
 
 void CLG_log_str(CLG_LogType *lg,

intern/clog/clog.c

@@ -81,6 +81,8 @@ typedef struct CLG_IDFilter {
 typedef struct CLogContext {
   /** Single linked list of types.  */
   CLG_LogType *types;
+  /** Single linked list of references.  */
+  CLG_LogRef *refs;
 #ifdef WITH_CLOG_PTHREADS
   pthread_mutex_t types_lock;
 #endif
@@ -673,6 +675,12 @@ static void CLG_ctx_free(CLogContext *ctx)
     MEM_freeN(item);
   }
 
+  while (ctx->refs != NULL) {
+    CLG_LogRef *item = ctx->refs;
+    ctx->refs = item->next;
+    item->type = NULL;
+  }
+
   for (uint i = 0; i < 2; i++) {
     while (ctx->filters[i] != NULL) {
       CLG_IDFilter *item = ctx->filters[i];
@@ -769,6 +777,10 @@ void CLG_logref_init(CLG_LogRef *clg_ref)
   pthread_mutex_lock(&g_ctx->types_lock);
 #endif
   if (clg_ref->type == NULL) {
+    /* Add to the refs list so we can NULL the pointers to 'type' when CLG_exit() is called. */
+    clg_ref->next = g_ctx->refs;
+    g_ctx->refs = clg_ref;
+
     CLG_LogType *clg_ty = clg_ctx_type_find_by_name(g_ctx, clg_ref->identifier);
     if (clg_ty == NULL) {
       clg_ty = clg_ctx_type_register(g_ctx, clg_ref->identifier);

intern/cycles/blender/addon/presets.py

@@ -41,6 +41,9 @@ class AddPresetIntegrator(AddPresetBase, Operator):
         "cycles.caustics_reflective",
         "cycles.caustics_refractive",
         "cycles.blur_glossy"
+        "cycles.use_fast_gi"
+        "cycles.ao_bounces"
+        "cycles.ao_bounces_render"
     ]
 
     preset_subdir = "cycles/integrator"

intern/cycles/blender/addon/properties.py

@@ -801,17 +801,22 @@ class CyclesRenderSettings(bpy.types.PropertyGroup):
         items=enum_texture_limit
     )
 
+    use_fast_gi: BoolProperty(
+        name="Fast GI Approximation",
+        description="Approximate diffuse indirect light with background tinted ambient occlusion. This provides fast alternative to full global illumination, for interactive viewport rendering or final renders with reduced quality",
+        default=False,
+    )
     ao_bounces: IntProperty(
         name="AO Bounces",
-        default=0,
-        description="Approximate indirect light with background tinted ambient occlusion at the specified bounce, 0 disables this feature",
+        default=1,
+        description="After this number of light bounces, use approximate global illumination. 0 disables this feature",
         min=0, max=1024,
     )
 
     ao_bounces_render: IntProperty(
         name="AO Bounces Render",
-        default=0,
-        description="Approximate indirect light with background tinted ambient occlusion at the specified bounce, 0 disables this feature",
+        default=1,
+        description="After this number of light bounces, use approximate global illumination. 0 disables this feature",
         min=0, max=1024,
     )
 

intern/cycles/blender/addon/ui.py

@@ -526,6 +526,37 @@ class CYCLES_RENDER_PT_light_paths_caustics(CyclesButtonsPanel, Panel):
         col.prop(cscene, "caustics_refractive", text="Refractive")
 
 
+class CYCLES_RENDER_PT_light_paths_fast_gi(CyclesButtonsPanel, Panel):
+    bl_label = "Fast GI Approximation"
+    bl_options = {'DEFAULT_CLOSED'}
+    bl_parent_id = "CYCLES_RENDER_PT_light_paths"
+
+    def draw_header(self, context):
+        scene = context.scene
+        cscene = scene.cycles
+
+        self.layout.prop(cscene, "use_fast_gi", text="")
+
+    def draw(self, context):
+        scene = context.scene
+        cscene = scene.cycles
+        world = scene.world
+
+        layout = self.layout
+        layout.use_property_split = True
+        layout.use_property_decorate = False
+
+        col = layout.column(align=True)
+        col.prop(cscene, "ao_bounces", text="Viewport Bounces")
+        col.prop(cscene, "ao_bounces_render", text="Render Bounces")
+
+        if world:
+          light = world.light_settings
+          col = layout.column(align=True)
+          col.prop(light, "ao_factor", text="AO Factor")
+          col.prop(light, "distance", text="AO Distance")
+
+
 class CYCLES_RENDER_PT_motion_blur(CyclesButtonsPanel, Panel):
     bl_label = "Motion Blur"
     bl_options = {'DEFAULT_CLOSED'}
@@ -694,7 +725,7 @@ class CYCLES_RENDER_PT_performance_tiles(CyclesButtonsPanel, Panel):
         col.prop(cscene, "tile_order", text="Order")
 
         sub = col.column()
-        sub.active = not rd.use_save_buffers
+        sub.active = not rd.use_save_buffers and not cscene.use_adaptive_sampling
         sub.prop(cscene, "use_progressive_refine")
 
 
@@ -746,7 +777,7 @@ class CYCLES_RENDER_PT_performance_final_render(CyclesButtonsPanel, Panel):
         col = layout.column()
 
         col.prop(rd, "use_save_buffers")
-        col.prop(rd, "use_persistent_data", text="Persistent Images")
+        col.prop(rd, "use_persistent_data", text="Persistent Data")
 
 
 class CYCLES_RENDER_PT_performance_viewport(CyclesButtonsPanel, Panel):
@@ -1409,15 +1440,15 @@ class CYCLES_LIGHT_PT_nodes(CyclesButtonsPanel, Panel):
         panel_node_draw(layout, light, 'OUTPUT_LIGHT', 'Surface')
 
 
-class CYCLES_LIGHT_PT_spot(CyclesButtonsPanel, Panel):
-    bl_label = "Spot Shape"
+class CYCLES_LIGHT_PT_beam_shape(CyclesButtonsPanel, Panel):
+    bl_label = "Beam Shape"
     bl_parent_id = "CYCLES_LIGHT_PT_light"
     bl_context = "data"
 
     @classmethod
     def poll(cls, context):
-        light = context.light
-        return (light and light.type == 'SPOT') and CyclesButtonsPanel.poll(context)
+        if context.light.type in {'SPOT', 'AREA'}:
+            return context.light and CyclesButtonsPanel.poll(context)
 
     def draw(self, context):
         layout = self.layout
@@ -1425,9 +1456,12 @@ class CYCLES_LIGHT_PT_spot(CyclesButtonsPanel, Panel):
         layout.use_property_split = True
 
         col = layout.column()
-        col.prop(light, "spot_size", text="Size")
-        col.prop(light, "spot_blend", text="Blend", slider=True)
-        col.prop(light, "show_cone")
+        if light.type == 'SPOT':
+            col.prop(light, "spot_size", text="Spot Size")
+            col.prop(light, "spot_blend", text="Blend", slider=True)
+            col.prop(light, "show_cone")
+        elif light.type == 'AREA':
+            col.prop(light, "spread", text="Spread")
 
 
 class CYCLES_WORLD_PT_preview(CyclesButtonsPanel, Panel):
@@ -2038,7 +2072,6 @@ class CYCLES_RENDER_PT_simplify_viewport(CyclesButtonsPanel, Panel):
         col.prop(rd, "simplify_subdivision", text="Max Subdivision")
         col.prop(rd, "simplify_child_particles", text="Child Particles")
         col.prop(cscene, "texture_limit", text="Texture Limit")
-        col.prop(cscene, "ao_bounces", text="AO Bounces")
         col.prop(rd, "simplify_volumes", text="Volume Resolution")
 
 
@@ -2064,7 +2097,6 @@ class CYCLES_RENDER_PT_simplify_render(CyclesButtonsPanel, Panel):
         col.prop(rd, "simplify_subdivision_render", text="Max Subdivision")
         col.prop(rd, "simplify_child_particles_render", text="Child Particles")
         col.prop(cscene, "texture_limit_render", text="Texture Limit")
-        col.prop(cscene, "ao_bounces_render", text="AO Bounces")
 
 
 class CYCLES_RENDER_PT_simplify_culling(CyclesButtonsPanel, Panel):
@@ -2242,6 +2274,7 @@ classes = (
     CYCLES_RENDER_PT_light_paths_max_bounces,
     CYCLES_RENDER_PT_light_paths_clamping,
     CYCLES_RENDER_PT_light_paths_caustics,
+    CYCLES_RENDER_PT_light_paths_fast_gi,
     CYCLES_RENDER_PT_volumes,
     CYCLES_RENDER_PT_subdivision,
     CYCLES_RENDER_PT_hair,
@@ -2284,7 +2317,7 @@ classes = (
     CYCLES_LIGHT_PT_preview,
     CYCLES_LIGHT_PT_light,
     CYCLES_LIGHT_PT_nodes,
-    CYCLES_LIGHT_PT_spot,
+    CYCLES_LIGHT_PT_beam_shape,
     CYCLES_WORLD_PT_preview,
     CYCLES_WORLD_PT_surface,
     CYCLES_WORLD_PT_volume,
@@ -2314,7 +2347,7 @@ classes = (
     node_panel(CYCLES_WORLD_PT_settings_surface),
     node_panel(CYCLES_WORLD_PT_settings_volume),
     node_panel(CYCLES_LIGHT_PT_light),
-    node_panel(CYCLES_LIGHT_PT_spot),
+    node_panel(CYCLES_LIGHT_PT_beam_shape)
 )
 
 

intern/cycles/blender/addon/version_update.py

@@ -217,6 +217,18 @@ def do_versions(self):
                         baov.name = caov.get("name", "AOV")
                         baov.type = "COLOR" if caov.get("type", 1) == 1 else "VALUE"
 
+            if version <= (2, 93, 16):
+                cscene = scene.cycles
+                ao_bounces = cscene.get("ao_bounces", 0)
+                ao_bounces_render = cscene.get("ao_bounces_render", 0)
+                if scene.render.use_simplify and (ao_bounces or ao_bounces_render):
+                    cscene.use_fast_gi = True
+                    cscene.ao_bounces = ao_bounces
+                    cscene.ao_bounces_render = ao_bounces_render
+                else:
+                    cscene.ao_bounces = 1
+                    cscene.ao_bounces_render = 1
+
         # Lamps
         for light in bpy.data.lights:
             if light.library not in libraries:

intern/cycles/blender/blender_light.cpp

@@ -82,6 +82,7 @@ void BlenderSync::sync_light(BL::Object &b_parent,
       light->set_axisu(transform_get_column(&tfm, 0));
       light->set_axisv(transform_get_column(&tfm, 1));
       light->set_sizeu(b_area_light.size());
+      light->set_spread(b_area_light.spread());
       switch (b_area_light.shape()) {
         case BL::AreaLight::shape_SQUARE:
           light->set_sizev(light->get_sizeu());

intern/cycles/blender/blender_object.cpp

@@ -96,7 +96,49 @@ bool BlenderSync::object_is_light(BL::Object &b_ob)
   return (b_ob_data && b_ob_data.is_a(&RNA_Light));
 }
 
-/* Object */
+void BlenderSync::sync_object_motion_init(BL::Object &b_parent, BL::Object &b_ob, Object *object)
+{
+  /* Initialize motion blur for object, detecting if it's enabled and creating motion
+   * steps array if so. */
+  array<Transform> motion;
+  object->set_motion(motion);
+
+  Scene::MotionType need_motion = scene->need_motion();
+  if (need_motion == Scene::MOTION_NONE || !object->get_geometry()) {
+    return;
+  }
+
+  Geometry *geom = object->get_geometry();
+  geom->set_use_motion_blur(false);
+  geom->set_motion_steps(0);
+
+  uint motion_steps;
+
+  if (need_motion == Scene::MOTION_BLUR) {
+    motion_steps = object_motion_steps(b_parent, b_ob, Object::MAX_MOTION_STEPS);
+    geom->set_motion_steps(motion_steps);
+    if (motion_steps && object_use_deform_motion(b_parent, b_ob)) {
+      geom->set_use_motion_blur(true);
+    }
+  }
+  else {
+    motion_steps = 3;
+    geom->set_motion_steps(motion_steps);
+  }
+
+  motion.resize(motion_steps, transform_empty());
+
+  if (motion_steps) {
+    motion[motion_steps / 2] = object->get_tfm();
+
+    /* update motion socket before trying to access object->motion_time */
+    object->set_motion(motion);
+
+    for (size_t step = 0; step < motion_steps; step++) {
+      motion_times.insert(object->motion_time(step));
+    }
+  }
+}
 
 Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
                                  BL::ViewLayer &b_view_layer,
@@ -219,10 +261,8 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
   }
 
   /* test if we need to sync */
-  bool object_updated = false;
-
-  if (object_map.add_or_update(&object, b_ob, b_parent, key))
-    object_updated = true;
+  bool object_updated = object_map.add_or_update(&object, b_ob, b_parent, key) ||
+                        (tfm != object->get_tfm());
 
   /* mesh sync */
   /* b_ob is owned by the iterator and will go out of scope at the end of the block.
@@ -271,49 +311,11 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
    * transform comparison should not be needed, but duplis don't work perfect
    * in the depsgraph and may not signal changes, so this is a workaround */
   if (object->is_modified() || object_updated ||
-      (object->get_geometry() && object->get_geometry()->is_modified()) ||
-      tfm != object->get_tfm()) {
+      (object->get_geometry() && object->get_geometry()->is_modified())) {
     object->name = b_ob.name().c_str();
     object->set_pass_id(b_ob.pass_index());
     object->set_color(get_float3(b_ob.color()));
     object->set_tfm(tfm);
-    array<Transform> motion;
-    object->set_motion(motion);
-
-    /* motion blur */
-    Scene::MotionType need_motion = scene->need_motion();
-    if (need_motion != Scene::MOTION_NONE && object->get_geometry()) {
-      Geometry *geom = object->get_geometry();
-      geom->set_use_motion_blur(false);
-      geom->set_motion_steps(0);
-
-      uint motion_steps;
-
-      if (need_motion == Scene::MOTION_BLUR) {
-        motion_steps = object_motion_steps(b_parent, b_ob, Object::MAX_MOTION_STEPS);
-        geom->set_motion_steps(motion_steps);
-        if (motion_steps && object_use_deform_motion(b_parent, b_ob)) {
-          geom->set_use_motion_blur(true);
-        }
-      }
-      else {
-        motion_steps = 3;
-        geom->set_motion_steps(motion_steps);
-      }
-
-      motion.resize(motion_steps, transform_empty());
-
-      if (motion_steps) {
-        motion[motion_steps / 2] = tfm;
-
-        /* update motion socket before trying to access object->motion_time */
-        object->set_motion(motion);
-
-        for (size_t step = 0; step < motion_steps; step++) {
-          motion_times.insert(object->motion_time(step));
-        }
-      }
-    }
 
     /* dupli texture coordinates and random_id */
     if (is_instance) {
@@ -331,6 +333,8 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
     object->tag_update(scene);
   }
 
+  sync_object_motion_init(b_parent, b_ob, object);
+
   if (is_instance) {
     /* Sync possible particle data. */
     sync_dupli_particle(b_parent, b_instance, object);
@@ -611,7 +615,7 @@ void BlenderSync::sync_motion(BL::RenderSettings &b_render,
     if (b_cam) {
       sync_camera_motion(b_render, b_cam, width, height, 0.0f);
     }
-    sync_objects(b_depsgraph, b_v3d, 0.0f);
+    sync_objects(b_depsgraph, b_v3d);
   }
 
   /* Insert motion times from camera. Motion times from other objects

intern/cycles/blender/blender_python.cpp

@@ -35,6 +35,7 @@
 #include "util/util_path.h"
 #include "util/util_string.h"
 #include "util/util_task.h"
+#include "util/util_tbb.h"
 #include "util/util_types.h"
 
 #ifdef WITH_OSL
@@ -288,9 +289,11 @@ static PyObject *render_func(PyObject * /*self*/, PyObject *args)
   RNA_pointer_create(NULL, &RNA_Depsgraph, (ID *)PyLong_AsVoidPtr(pydepsgraph), &depsgraphptr);
   BL::Depsgraph b_depsgraph(depsgraphptr);
 
+  /* Allow Blender to execute other Python scripts, and isolate TBB tasks so we
+   * don't get deadlocks with Blender threads accessing shared data like images. */
   python_thread_state_save(&session->python_thread_state);
 
-  session->render(b_depsgraph);
+  tbb::this_task_arena::isolate([&] { session->render(b_depsgraph); });
 
   python_thread_state_restore(&session->python_thread_state);
 
@@ -327,7 +330,8 @@ static PyObject *bake_func(PyObject * /*self*/, PyObject *args)
 
   python_thread_state_save(&session->python_thread_state);
 
-  session->bake(b_depsgraph, b_object, pass_type, pass_filter, width, height);
+  tbb::this_task_arena::isolate(
+      [&] { session->bake(b_depsgraph, b_object, pass_type, pass_filter, width, height); });
 
   python_thread_state_restore(&session->python_thread_state);
 
@@ -373,7 +377,7 @@ static PyObject *reset_func(PyObject * /*self*/, PyObject *args)
 
   python_thread_state_save(&session->python_thread_state);
 
-  session->reset_session(b_data, b_depsgraph);
+  tbb::this_task_arena::isolate([&] { session->reset_session(b_data, b_depsgraph); });
 
   python_thread_state_restore(&session->python_thread_state);
 
@@ -395,7 +399,7 @@ static PyObject *sync_func(PyObject * /*self*/, PyObject *args)
 
   python_thread_state_save(&session->python_thread_state);
 
-  session->synchronize(b_depsgraph);
+  tbb::this_task_arena::isolate([&] { session->synchronize(b_depsgraph); });
 
   python_thread_state_restore(&session->python_thread_state);
 

intern/cycles/blender/blender_session.cpp

@@ -143,12 +143,6 @@ void BlenderSession::create_session()
 
   session->scene = scene;
 
-  /* There is no single depsgraph to use for the entire render.
-   * So we need to handle this differently.
-   *
-   * We could loop over the final render result render layers in pipeline and keep Cycles unaware
-   * of multiple layers, or perhaps move syncing further down in the pipeline.
-   */
   /* create sync */
   sync = new BlenderSync(b_engine, b_data, b_scene, scene, !background, session->progress);
   BL::Object b_camera_override(b_engine.camera_override());
@@ -213,7 +207,7 @@ void BlenderSession::reset_session(BL::BlendData &b_data, BL::Depsgraph &b_depsg
   SceneParams scene_params = BlenderSync::get_scene_params(b_scene, background);
 
   if (scene->params.modified(scene_params) || session->params.modified(session_params) ||
-      !scene_params.persistent_data) {
+      !this->b_render.use_persistent_data()) {
     /* if scene or session parameters changed, it's easier to simply re-create
      * them rather than trying to distinguish which settings need to be updated
      */
@@ -225,7 +219,6 @@ void BlenderSession::reset_session(BL::BlendData &b_data, BL::Depsgraph &b_depsg
   }
 
   session->progress.reset();
-  scene->reset();
 
   session->tile_manager.set_tile_order(session_params.tile_order);
 
@@ -234,12 +227,18 @@ void BlenderSession::reset_session(BL::BlendData &b_data, BL::Depsgraph &b_depsg
    */
   session->stats.mem_peak = session->stats.mem_used;
 
-  /* There is no single depsgraph to use for the entire render.
-   * See note on create_session().
-   */
-  /* sync object should be re-created */
-  delete sync;
-  sync = new BlenderSync(b_engine, b_data, b_scene, scene, !background, session->progress);
+  if (is_new_session) {
+    /* Sync object should be re-created for new scene. */
+    delete sync;
+    sync = new BlenderSync(b_engine, b_data, b_scene, scene, !background, session->progress);
+  }
+  else {
+    /* Sync recalculations to do just the required updates. */
+    sync->sync_recalc(b_depsgraph, b_v3d);
+  }
+
+  BL::Object b_camera_override(b_engine.camera_override());
+  sync->sync_camera(b_render, b_camera_override, width, height, "");
 
   BL::SpaceView3D b_null_space_view3d(PointerRNA_NULL);
   BL::RegionView3D b_null_region_view3d(PointerRNA_NULL);
@@ -502,7 +501,7 @@ void BlenderSession::render(BL::Depsgraph &b_depsgraph_)
 
   /* Compute render passes and film settings. */
   vector<Pass> passes = sync->sync_render_passes(
-      b_rlay, b_view_layer, session_params.adaptive_sampling, session_params.denoising);
+      b_scene, b_rlay, b_view_layer, session_params.adaptive_sampling, session_params.denoising);
 
   /* Set buffer params, using film settings from sync_render_passes. */
   buffer_params.passes = passes;
@@ -598,18 +597,6 @@ void BlenderSession::render(BL::Depsgraph &b_depsgraph_)
   /* clear callback */
   session->write_render_tile_cb = function_null;
   session->update_render_tile_cb = function_null;
-
-  /* TODO: find a way to clear this data for persistent data render */
-#if 0
-  /* free all memory used (host and device), so we wouldn't leave render
-   * engine with extra memory allocated
-   */
-
-  session->device_free();
-
-  delete sync;
-  sync = NULL;
-#endif
 }
 
 static int bake_pass_filter_get(const int pass_filter)

intern/cycles/blender/blender_shader.cpp

@@ -1553,13 +1553,9 @@ void BlenderSync::sync_lights(BL::Depsgraph &b_depsgraph, bool update_all)
 void BlenderSync::sync_shaders(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d)
 {
   /* for auto refresh images */
-  bool auto_refresh_update = false;
-
-  if (preview) {
-    ImageManager *image_manager = scene->image_manager;
-    int frame = b_scene.frame_current();
-    auto_refresh_update = image_manager->set_animation_frame_update(frame);
-  }
+  ImageManager *image_manager = scene->image_manager;
+  const int frame = b_scene.frame_current();
+  const bool auto_refresh_update = image_manager->set_animation_frame_update(frame);
 
   shader_map.pre_sync();
 

intern/cycles/blender/blender_sync.cpp

@@ -211,9 +211,11 @@ void BlenderSync::sync_recalc(BL::Depsgraph &b_depsgraph, BL::SpaceView3D &b_v3d
     }
   }
 
-  BlenderViewportParameters new_viewport_parameters(b_v3d);
-  if (viewport_parameters.modified(new_viewport_parameters)) {
-    world_recalc = true;
+  if (b_v3d) {
+    BlenderViewportParameters new_viewport_parameters(b_v3d);
+    if (viewport_parameters.modified(new_viewport_parameters)) {
+      world_recalc = true;
+    }
   }
 }
 
@@ -358,7 +360,7 @@ void BlenderSync::sync_integrator()
 
   integrator->set_adaptive_min_samples(adaptive_min_samples);
 
-  if (b_scene.render().use_simplify()) {
+  if (get_boolean(cscene, "use_fast_gi")) {
     if (preview) {
       integrator->set_ao_bounces(get_int(cscene, "ao_bounces"));
     }
@@ -567,7 +569,8 @@ int BlenderSync::get_denoising_pass(BL::RenderPass &b_pass)
   return -1;
 }
 
-vector<Pass> BlenderSync::sync_render_passes(BL::RenderLayer &b_rlay,
+vector<Pass> BlenderSync::sync_render_passes(BL::Scene &b_scene,
+                                             BL::RenderLayer &b_rlay,
                                              BL::ViewLayer &b_view_layer,
                                              bool adaptive_sampling,
                                              const DenoiseParams &denoising)
@@ -578,7 +581,7 @@ vector<Pass> BlenderSync::sync_render_passes(BL::RenderLayer &b_rlay,
   for (BL::RenderPass &b_pass : b_rlay.passes) {
     PassType pass_type = get_pass_type(b_pass);
 
-    if (pass_type == PASS_MOTION && scene->integrator->get_motion_blur())
+    if (pass_type == PASS_MOTION && b_scene.render().use_motion_blur())
       continue;
     if (pass_type != PASS_NONE)
       Pass::add(pass_type, passes, b_pass.name().c_str());
@@ -736,12 +739,18 @@ void BlenderSync::free_data_after_sync(BL::Depsgraph &b_depsgraph)
    * caches to be releases from blender side in order to reduce peak memory
    * footprint during synchronization process.
    */
+
   const bool is_interface_locked = b_engine.render() && b_engine.render().use_lock_interface();
-  const bool can_free_caches = (BlenderSession::headless || is_interface_locked) &&
-                               /* Baking re-uses the depsgraph multiple times, clearing crashes
-                                * reading un-evaluated mesh data which isn't aligned with the
-                                * geometry we're baking, see T71012. */
-                               !scene->bake_manager->get_baking();
+  const bool is_persistent_data = b_engine.render() && b_engine.render().use_persistent_data();
+  const bool can_free_caches =
+      (BlenderSession::headless || is_interface_locked) &&
+      /* Baking re-uses the depsgraph multiple times, clearing crashes
+       * reading un-evaluated mesh data which isn't aligned with the
+       * geometry we're baking, see T71012. */
+      !scene->bake_manager->get_baking() &&
+      /* Persistent data must main caches for performance and correctness. */
+      !is_persistent_data;
+
   if (!can_free_caches) {
     return;
   }
@@ -757,7 +766,6 @@ void BlenderSync::free_data_after_sync(BL::Depsgraph &b_depsgraph)
 
 SceneParams BlenderSync::get_scene_params(BL::Scene &b_scene, bool background)
 {
-  BL::RenderSettings r = b_scene.render();
   SceneParams params;
   PointerRNA cscene = RNA_pointer_get(&b_scene.ptr, "cycles");
   const bool shadingsystem = RNA_boolean_get(&cscene, "shading_system");
@@ -781,11 +789,6 @@ SceneParams BlenderSync::get_scene_params(BL::Scene &b_scene, bool background)
   params.hair_shape = (CurveShapeType)get_enum(
       csscene, "shape", CURVE_NUM_SHAPE_TYPES, CURVE_THICK);
 
-  if (background && params.shadingsystem != SHADINGSYSTEM_OSL)
-    params.persistent_data = r.use_persistent_data();
-  else
-    params.persistent_data = false;
-
   int texture_limit;
   if (background) {
     texture_limit = RNA_enum_get(&cscene, "texture_limit_render");
@@ -872,6 +875,9 @@ SessionParams BlenderSync::get_session_params(BL::RenderEngine &b_engine,
   /* Clamp samples. */
   params.samples = min(params.samples, Integrator::MAX_SAMPLES);
 
+  /* Adaptive sampling. */
+  params.adaptive_sampling = RNA_boolean_get(&cscene, "use_adaptive_sampling");
+
   /* tiles */
   const bool is_cpu = (params.device.type == DEVICE_CPU);
   if (!is_cpu && !background) {
@@ -924,7 +930,7 @@ SessionParams BlenderSync::get_session_params(BL::RenderEngine &b_engine,
   BL::RenderSettings b_r = b_scene.render();
   params.progressive_refine = b_engine.is_preview() ||
                               get_boolean(cscene, "use_progressive_refine");
-  if (b_r.use_save_buffers())
+  if (b_r.use_save_buffers() || params.adaptive_sampling)
     params.progressive_refine = false;
 
   if (background) {
@@ -960,8 +966,6 @@ SessionParams BlenderSync::get_session_params(BL::RenderEngine &b_engine,
   params.use_profiling = params.device.has_profiling && !b_engine.is_preview() && background &&
                          BlenderSession::print_render_stats;
 
-  params.adaptive_sampling = RNA_boolean_get(&cscene, "use_adaptive_sampling");
-
   return params;
 }
 

intern/cycles/blender/blender_sync.h

@@ -74,7 +74,8 @@ class BlenderSync {
                  int height,
                  void **python_thread_state);
   void sync_view_layer(BL::SpaceView3D &b_v3d, BL::ViewLayer &b_view_layer);
-  vector<Pass> sync_render_passes(BL::RenderLayer &b_render_layer,
+  vector<Pass> sync_render_passes(BL::Scene &b_scene,
+                                  BL::RenderLayer &b_render_layer,
                                   BL::ViewLayer &b_view_layer,
                                   bool adaptive_sampling,
                                   const DenoiseParams &denoising);
@@ -149,6 +150,7 @@ class BlenderSync {
                       BlenderObjectCulling &culling,
                       bool *use_portal,
                       TaskPool *geom_task_pool);
+  void sync_object_motion_init(BL::Object &b_parent, BL::Object &b_ob, Object *object);
 
   bool sync_object_attributes(BL::DepsgraphObjectInstance &b_instance, Object *object);
 

intern/cycles/blender/blender_volume.cpp

@@ -26,7 +26,7 @@
 #ifdef WITH_OPENVDB
 #  include <openvdb/openvdb.h>
 openvdb::GridBase::ConstPtr BKE_volume_grid_openvdb_for_read(const struct Volume *volume,
-                                                             struct VolumeGrid *grid);
+                                                             const struct VolumeGrid *grid);
 #endif
 
 CCL_NAMESPACE_BEGIN
@@ -227,7 +227,7 @@ class BlenderVolumeLoader : public VDBImageLoader {
         const bool unload = !b_volume_grid.is_loaded();
 
         ::Volume *volume = (::Volume *)b_volume.ptr.data;
-        VolumeGrid *volume_grid = (VolumeGrid *)b_volume_grid.ptr.data;
+        const VolumeGrid *volume_grid = (VolumeGrid *)b_volume_grid.ptr.data;
         grid = BKE_volume_grid_openvdb_for_read(volume, volume_grid);
 
         if (unload) {

intern/cycles/bvh/bvh_optix.cpp

@@ -17,6 +17,8 @@
 
 #ifdef WITH_OPTIX
 
+#  include "device/device.h"
+
 #  include "bvh/bvh_optix.h"
 
 CCL_NAMESPACE_BEGIN
@@ -26,6 +28,7 @@ BVHOptiX::BVHOptiX(const BVHParams &params_,
                    const vector<Object *> &objects_,
                    Device *device)
     : BVH(params_, geometry_, objects_),
+      device(device),
       traversable_handle(0),
       as_data(device, params_.top_level ? "optix tlas" : "optix blas", false),
       motion_transform_data(device, "optix motion transform", false)
@@ -34,7 +37,9 @@ BVHOptiX::BVHOptiX(const BVHParams &params_,
 
 BVHOptiX::~BVHOptiX()
 {
-  // Acceleration structure memory is freed via the 'as_data' destructor
+  // Acceleration structure memory is delayed freed on device, since deleting the
+  // BVH may happen while still being used for rendering.
+  device->release_optix_bvh(this);
 }
 
 CCL_NAMESPACE_END

intern/cycles/bvh/bvh_optix.h

@@ -28,6 +28,7 @@ CCL_NAMESPACE_BEGIN
 
 class BVHOptiX : public BVH {
  public:
+  Device *device;
   uint64_t traversable_handle;
   device_only_memory<char> as_data;
   device_only_memory<char> motion_transform_data;

intern/cycles/device/device.h

@@ -61,7 +61,6 @@ enum DeviceTypeMask {
 };
 
 enum DeviceKernelStatus {
-  DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL = 0,
   DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE,
   DEVICE_KERNEL_USING_FEATURE_KERNEL,
   DEVICE_KERNEL_FEATURE_KERNEL_INVALID,
@@ -427,6 +426,9 @@ class Device {
   /* acceleration structure building */
   virtual void build_bvh(BVH *bvh, Progress &progress, bool refit);
 
+  /* OptiX specific destructor. */
+  virtual void release_optix_bvh(BVH *){};
+
 #ifdef WITH_NETWORK
   /* networking */
   void server_run();

intern/cycles/device/device_memory.cpp

@@ -35,10 +35,54 @@ device_memory::device_memory(Device *device, const char *name, MemoryType type)
       device_pointer(0),
       host_pointer(0),
       shared_pointer(0),
-      shared_counter(0)
+      shared_counter(0),
+      original_device_ptr(0),
+      original_device_size(0),
+      original_device(0),
+      need_realloc_(false),
+      modified(false)
 {
 }
 
+device_memory::device_memory(device_memory &&other) noexcept
+    : data_type(other.data_type),
+      data_elements(other.data_elements),
+      data_size(other.data_size),
+      device_size(other.device_size),
+      data_width(other.data_width),
+      data_height(other.data_height),
+      data_depth(other.data_depth),
+      type(other.type),
+      name(other.name),
+      device(other.device),
+      device_pointer(other.device_pointer),
+      host_pointer(other.host_pointer),
+      shared_pointer(other.shared_pointer),
+      shared_counter(other.shared_counter),
+      original_device_ptr(other.original_device_ptr),
+      original_device_size(other.original_device_size),
+      original_device(other.original_device),
+      need_realloc_(other.need_realloc_),
+      modified(other.modified)
+{
+  other.data_elements = 0;
+  other.data_size = 0;
+  other.device_size = 0;
+  other.data_width = 0;
+  other.data_height = 0;
+  other.data_depth = 0;
+  other.device = 0;
+  other.device_pointer = 0;
+  other.host_pointer = 0;
+  other.shared_pointer = 0;
+  other.shared_counter = 0;
+  other.original_device_ptr = 0;
+  other.original_device_size = 0;
+  other.original_device = 0;
+  other.need_realloc_ = false;
+  other.modified = false;
+}
+
 device_memory::~device_memory()
 {
   assert(shared_pointer == 0);

intern/cycles/device/device_memory.h

@@ -238,6 +238,7 @@ class device_memory {
 
   /* Only create through subclasses. */
   device_memory(Device *device, const char *name, MemoryType type);
+  device_memory(device_memory &&other) noexcept;
 
   /* No copying allowed. */
   device_memory(const device_memory &) = delete;
@@ -277,6 +278,10 @@ template<typename T> class device_only_memory : public device_memory {
     data_elements = max(device_type_traits<T>::num_elements, 1);
   }
 
+  device_only_memory(device_only_memory &&other) noexcept : device_memory(std::move(other))
+  {
+  }
+
   virtual ~device_only_memory()
   {
     free();

intern/cycles/device/device_multi.cpp

@@ -232,10 +232,6 @@ class MultiDevice : public Device {
     foreach (SubDevice &sub, devices) {
       DeviceKernelStatus subresult = sub.device->get_active_kernel_switch_state();
       switch (subresult) {
-        case DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL:
-          result = subresult;
-          break;
-
         case DEVICE_KERNEL_FEATURE_KERNEL_INVALID:
         case DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE:
           return subresult;

intern/cycles/device/device_optix.cpp

@@ -193,6 +193,9 @@ class OptiXDevice : public CUDADevice {
   device_only_memory<unsigned char> denoiser_state;
   int denoiser_input_passes = 0;
 
+  vector<device_only_memory<char>> delayed_free_bvh_memory;
+  thread_mutex delayed_free_bvh_mutex;
+
  public:
   OptiXDevice(DeviceInfo &info_, Stats &stats_, Profiler &profiler_, bool background_)
       : CUDADevice(info_, stats_, profiler_, background_),
@@ -258,6 +261,8 @@ class OptiXDevice : public CUDADevice {
     // Make CUDA context current
     const CUDAContextScope scope(cuContext);
 
+    free_bvh_memory_delayed();
+
     sbt_data.free();
     texture_info.free();
     launch_params.free();
@@ -362,7 +367,7 @@ class OptiXDevice : public CUDADevice {
       }
     }
 
-    OptixModuleCompileOptions module_options;
+    OptixModuleCompileOptions module_options = {};
     module_options.maxRegisterCount = 0;  // Do not set an explicit register limit
 #  ifdef WITH_CYCLES_DEBUG
     module_options.optLevel = OPTIX_COMPILE_OPTIMIZATION_LEVEL_0;
@@ -377,7 +382,7 @@ class OptiXDevice : public CUDADevice {
     module_options.numBoundValues = 0;
 #  endif
 
-    OptixPipelineCompileOptions pipeline_options;
+    OptixPipelineCompileOptions pipeline_options = {};
     // Default to no motion blur and two-level graph, since it is the fastest option
     pipeline_options.usesMotionBlur = false;
     pipeline_options.traversableGraphFlags =
@@ -477,7 +482,7 @@ class OptiXDevice : public CUDADevice {
 
 #  if OPTIX_ABI_VERSION >= 36
       if (DebugFlags().optix.curves_api && requested_features.use_hair_thick) {
-        OptixBuiltinISOptions builtin_options;
+        OptixBuiltinISOptions builtin_options = {};
         builtin_options.builtinISModuleType = OPTIX_PRIMITIVE_TYPE_ROUND_CUBIC_BSPLINE;
         builtin_options.usesMotionBlur = false;
 
@@ -571,7 +576,7 @@ class OptiXDevice : public CUDADevice {
                          stack_size[PG_HITS_MOTION].cssIS + stack_size[PG_HITS_MOTION].cssAH);
 #  endif
 
-    OptixPipelineLinkOptions link_options;
+    OptixPipelineLinkOptions link_options = {};
     link_options.maxTraceDepth = 1;
 #  ifdef WITH_CYCLES_DEBUG
     link_options.debugLevel = OPTIX_COMPILE_DEBUG_LEVEL_FULL;
@@ -953,7 +958,7 @@ class OptiXDevice : public CUDADevice {
         }
 
         // Create OptiX denoiser handle on demand when it is first used
-        OptixDenoiserOptions denoiser_options;
+        OptixDenoiserOptions denoiser_options = {};
         assert(task.denoising.input_passes >= 1 && task.denoising.input_passes <= 3);
         denoiser_options.inputKind = static_cast<OptixDenoiserInputKind>(
             OPTIX_DENOISER_INPUT_RGB + (task.denoising.input_passes - 1));
@@ -1157,7 +1162,7 @@ class OptiXDevice : public CUDADevice {
 
     // Compute memory usage
     OptixAccelBufferSizes sizes = {};
-    OptixAccelBuildOptions options;
+    OptixAccelBuildOptions options = {};
     options.operation = operation;
     if (background) {
       // Prefer best performance and lowest memory consumption in background
@@ -1195,7 +1200,7 @@ class OptiXDevice : public CUDADevice {
     }
 
     // Finally build the acceleration structure
-    OptixAccelEmitDesc compacted_size_prop;
+    OptixAccelEmitDesc compacted_size_prop = {};
     compacted_size_prop.type = OPTIX_PROPERTY_TYPE_COMPACTED_SIZE;
     // A tiny space was allocated for this property at the end of the temporary buffer above
     // Make sure this pointer is 8-byte aligned
@@ -1265,6 +1270,8 @@ class OptiXDevice : public CUDADevice {
       return;
     }
 
+    free_bvh_memory_delayed();
+
     BVHOptiX *const bvh_optix = static_cast<BVHOptiX *>(bvh);
 
     progress.set_substatus("Building OptiX acceleration structure");
@@ -1735,6 +1742,24 @@ class OptiXDevice : public CUDADevice {
     }
   }
 
+  void release_optix_bvh(BVH *bvh) override
+  {
+    thread_scoped_lock lock(delayed_free_bvh_mutex);
+    /* Do delayed free of BVH memory, since geometry holding BVH might be deleted
+     * while GPU is still rendering. */
+    BVHOptiX *const bvh_optix = static_cast<BVHOptiX *>(bvh);
+
+    delayed_free_bvh_memory.emplace_back(std::move(bvh_optix->as_data));
+    delayed_free_bvh_memory.emplace_back(std::move(bvh_optix->motion_transform_data));
+    bvh_optix->traversable_handle = 0;
+  }
+
+  void free_bvh_memory_delayed()
+  {
+    thread_scoped_lock lock(delayed_free_bvh_mutex);
+    delayed_free_bvh_memory.free_memory();
+  }
+
   void const_copy_to(const char *name, void *host, size_t size) override
   {
     // Set constant memory for CUDA module

intern/cycles/device/opencl/device_opencl.h

@@ -269,7 +269,6 @@ class OpenCLDevice : public Device {
   cl_device_id cdDevice;
   cl_int ciErr;
   int device_num;
-  bool use_preview_kernels;
 
   class OpenCLProgram {
    public:
@@ -369,8 +368,7 @@ class OpenCLDevice : public Device {
     /* Load the kernels and put the created kernels in the given
      * `programs` parameter. */
     void load_kernels(vector<OpenCLProgram *> &programs,
-                      const DeviceRequestedFeatures &requested_features,
-                      bool is_preview = false);
+                      const DeviceRequestedFeatures &requested_features);
   };
 
   DeviceSplitKernel *split_kernel;
@@ -382,7 +380,6 @@ class OpenCLDevice : public Device {
   OpenCLProgram denoising_program;
 
   OpenCLSplitPrograms kernel_programs;
-  OpenCLSplitPrograms preview_programs;
 
   typedef map<string, device_vector<uchar> *> ConstMemMap;
   typedef map<string, device_ptr> MemMap;
@@ -412,7 +409,6 @@ class OpenCLDevice : public Device {
   string device_md5_hash(string kernel_custom_build_options = "");
   bool load_kernels(const DeviceRequestedFeatures &requested_features);
   void load_required_kernels(const DeviceRequestedFeatures &requested_features);
-  void load_preview_kernels();
 
   bool wait_for_availability(const DeviceRequestedFeatures &requested_features);
   DeviceKernelStatus get_active_kernel_switch_state();
@@ -422,8 +418,7 @@ class OpenCLDevice : public Device {
   /* Get the program file name to compile (*.cl) for the given kernel */
   const string get_opencl_program_filename(const string &kernel_name);
   string get_build_options(const DeviceRequestedFeatures &requested_features,
-                           const string &opencl_program_name,
-                           bool preview_kernel = false);
+                           const string &opencl_program_name);
   /* Enable the default features to reduce recompilation events */
   void enable_default_features(DeviceRequestedFeatures &features);
 

intern/cycles/device/opencl/device_opencl_impl.cpp

@@ -107,8 +107,7 @@ void OpenCLDevice::enable_default_features(DeviceRequestedFeatures &features)
 }
 
 string OpenCLDevice::get_build_options(const DeviceRequestedFeatures &requested_features,
-                                       const string &opencl_program_name,
-                                       bool preview_kernel)
+                                       const string &opencl_program_name)
 {
   /* first check for non-split kernel programs */
   if (opencl_program_name == "base" || opencl_program_name == "denoising") {
@@ -185,13 +184,7 @@ string OpenCLDevice::get_build_options(const DeviceRequestedFeatures &requested_
   enable_default_features(nofeatures);
 
   /* Add program specific optimized compile directives */
-  if (preview_kernel) {
-    DeviceRequestedFeatures preview_features;
-    preview_features.use_hair = true;
-    build_options += "-D__KERNEL_AO_PREVIEW__ ";
-    build_options += preview_features.get_build_options();
-  }
-  else if (opencl_program_name == "split_do_volume" && !requested_features.use_volume) {
+  if (opencl_program_name == "split_do_volume" && !requested_features.use_volume) {
     build_options += nofeatures.get_build_options();
   }
   else {
@@ -238,9 +231,7 @@ OpenCLDevice::OpenCLSplitPrograms::~OpenCLSplitPrograms()
 }
 
 void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
-    vector<OpenCLProgram *> &programs,
-    const DeviceRequestedFeatures &requested_features,
-    bool is_preview)
+    vector<OpenCLProgram *> &programs, const DeviceRequestedFeatures &requested_features)
 {
   if (!requested_features.use_baking) {
 #  define ADD_SPLIT_KERNEL_BUNDLE_PROGRAM(kernel_name) \
@@ -251,7 +242,7 @@ void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
         device, \
         program_name_##kernel_name, \
         "kernel_" #kernel_name ".cl", \
-        device->get_build_options(requested_features, program_name_##kernel_name, is_preview)); \
+        device->get_build_options(requested_features, program_name_##kernel_name)); \
     program_##kernel_name.add_kernel(ustring("path_trace_" #kernel_name)); \
     programs.push_back(&program_##kernel_name);
 
@@ -259,7 +250,7 @@ void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
     ADD_SPLIT_KERNEL_PROGRAM(subsurface_scatter);
     ADD_SPLIT_KERNEL_PROGRAM(direct_lighting);
     ADD_SPLIT_KERNEL_PROGRAM(indirect_background);
-    if (requested_features.use_volume || is_preview) {
+    if (requested_features.use_volume) {
       ADD_SPLIT_KERNEL_PROGRAM(do_volume);
     }
     ADD_SPLIT_KERNEL_PROGRAM(shader_eval);
@@ -274,7 +265,7 @@ void OpenCLDevice::OpenCLSplitPrograms::load_kernels(
         device,
         "split_bundle",
         "kernel_split_bundle.cl",
-        device->get_build_options(requested_features, "split_bundle", is_preview));
+        device->get_build_options(requested_features, "split_bundle"));
 
     ADD_SPLIT_KERNEL_BUNDLE_PROGRAM(data_init);
     ADD_SPLIT_KERNEL_BUNDLE_PROGRAM(state_buffer_size);
@@ -403,7 +394,7 @@ class OpenCLSplitKernel : public DeviceSplitKernel {
         device,
         program_name,
         device->get_opencl_program_filename(kernel_name),
-        device->get_build_options(requested_features, program_name, device->use_preview_kernels));
+        device->get_build_options(requested_features, program_name));
 
     kernel->program.add_kernel(ustring("path_trace_" + kernel_name));
     kernel->program.load();
@@ -569,6 +560,11 @@ class OpenCLSplitKernel : public DeviceSplitKernel {
     size_t num_elements = max_elements_for_max_buffer_size(kg, data, max_buffer_size);
     int2 global_size = make_int2(max(round_down((int)sqrt(num_elements), 64), 64),
                                  (int)sqrt(num_elements));
+
+    if (device->info.description.find("Intel") != string::npos) {
+      global_size = make_int2(min(512, global_size.x), min(512, global_size.y));
+    }
+
     VLOG(1) << "Global size: " << global_size << ".";
     return global_size;
   }
@@ -612,7 +608,6 @@ OpenCLDevice::OpenCLDevice(DeviceInfo &info, Stats &stats, Profiler &profiler, b
     : Device(info, stats, profiler, background),
       load_kernel_num_compiling(0),
       kernel_programs(this),
-      preview_programs(this),
       memory_manager(this),
       texture_info(this, "__texture_info", MEM_GLOBAL)
 {
@@ -622,7 +617,6 @@ OpenCLDevice::OpenCLDevice(DeviceInfo &info, Stats &stats, Profiler &profiler, b
   cqCommandQueue = NULL;
   device_initialized = false;
   textures_need_update = true;
-  use_preview_kernels = !background;
 
   vector<OpenCLPlatformDevice> usable_devices;
   OpenCLInfo::get_usable_devices(&usable_devices);
@@ -678,9 +672,6 @@ OpenCLDevice::OpenCLDevice(DeviceInfo &info, Stats &stats, Profiler &profiler, b
   device_initialized = true;
 
   split_kernel = new OpenCLSplitKernel(this);
-  if (use_preview_kernels) {
-    load_preview_kernels();
-  }
 }
 
 OpenCLDevice::~OpenCLDevice()
@@ -771,7 +762,7 @@ bool OpenCLDevice::load_kernels(const DeviceRequestedFeatures &requested_feature
   load_required_kernels(requested_features);
 
   vector<OpenCLProgram *> programs;
-  kernel_programs.load_kernels(programs, requested_features, false);
+  kernel_programs.load_kernels(programs, requested_features);
 
   if (!requested_features.use_baking && requested_features.use_denoising) {
     denoising_program = OpenCLProgram(
@@ -849,19 +840,6 @@ void OpenCLDevice::load_required_kernels(const DeviceRequestedFeatures &requeste
   }
 }
 
-void OpenCLDevice::load_preview_kernels()
-{
-  DeviceRequestedFeatures no_features;
-  vector<OpenCLProgram *> programs;
-  preview_programs.load_kernels(programs, no_features, true);
-
-  foreach (OpenCLProgram *program, programs) {
-    if (!program->load()) {
-      load_required_kernel_task_pool.push(function_bind(&OpenCLProgram::compile, program));
-    }
-  }
-}
-
 bool OpenCLDevice::wait_for_availability(const DeviceRequestedFeatures &requested_features)
 {
   if (requested_features.use_baking) {
@@ -869,59 +847,18 @@ bool OpenCLDevice::wait_for_availability(const DeviceRequestedFeatures &requeste
     return true;
   }
 
-  if (background) {
-    load_kernel_task_pool.wait_work();
-    use_preview_kernels = false;
-  }
-  else {
-    /* We use a device setting to determine to load preview kernels or not
-     * Better to check on device level than per kernel as mixing preview and
-     * non-preview kernels does not work due to different data types */
-    if (use_preview_kernels) {
-      use_preview_kernels = load_kernel_num_compiling.load() > 0;
-    }
-  }
+  load_kernel_task_pool.wait_work();
   return split_kernel->load_kernels(requested_features);
 }
 
 OpenCLDevice::OpenCLSplitPrograms *OpenCLDevice::get_split_programs()
 {
-  return use_preview_kernels ? &preview_programs : &kernel_programs;
+  return &kernel_programs;
 }
 
 DeviceKernelStatus OpenCLDevice::get_active_kernel_switch_state()
 {
-  /* Do not switch kernels for background renderings
-   * We do foreground rendering but use the preview kernels
-   * Check for the optimized kernels
-   *
-   * This works also the other way around, where we are using
-   * optimized kernels but new ones are being compiled due
-   * to other features that are needed */
-  if (background) {
-    /* The if-statements below would find the same result,
-     * But as the `finished` method uses a mutex we added
-     * this as an early exit */
-    return DEVICE_KERNEL_USING_FEATURE_KERNEL;
-  }
-
-  bool other_kernels_finished = load_kernel_num_compiling.load() == 0;
-  if (use_preview_kernels) {
-    if (other_kernels_finished) {
-      return DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE;
-    }
-    else {
-      return DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL;
-    }
-  }
-  else {
-    if (other_kernels_finished) {
-      return DEVICE_KERNEL_USING_FEATURE_KERNEL;
-    }
-    else {
-      return DEVICE_KERNEL_FEATURE_KERNEL_INVALID;
-    }
-  }
+  return DEVICE_KERNEL_USING_FEATURE_KERNEL;
 }
 
 void OpenCLDevice::mem_alloc(device_memory &mem)

intern/cycles/graph/node_type.h

@@ -148,16 +148,17 @@ struct NodeType {
 #define NODE_DECLARE \
   static const NodeType *get_node_type(); \
   template<typename T> static const NodeType *register_type(); \
-  static Node *create(const NodeType *type);
+  static Node *create(const NodeType *type); \
+  static const NodeType *node_type;
 
 #define NODE_DEFINE(structname) \
+  const NodeType *structname::node_type = structname::register_type<structname>(); \
   Node *structname::create(const NodeType *) \
   { \
     return new structname(); \
   } \
   const NodeType *structname::get_node_type() \
   { \
-    static const NodeType *node_type = register_type<structname>(); \
     return node_type; \
   } \
   template<typename T> const NodeType *structname::register_type()
@@ -169,6 +170,8 @@ struct NodeType {
 #define NODE_ABSTRACT_DEFINE(structname) \
   const NodeType *structname::get_node_base_type() \
   { \
+    /* Base types constructed in this getter to ensure correct initialization \
+     * order. Regular types are not so they are auto-registered for XML parsing. */ \
     static const NodeType *node_base_type = register_base_type<structname>(); \
     return node_base_type; \
   } \

intern/cycles/kernel/closure/alloc.h

@@ -57,14 +57,24 @@ ccl_device ccl_addr_space void *closure_alloc_extra(ShaderData *sd, int size)
 
 ccl_device_inline ShaderClosure *bsdf_alloc(ShaderData *sd, int size, float3 weight)
 {
-  ShaderClosure *sc = closure_alloc(sd, size, CLOSURE_NONE_ID, weight);
+  kernel_assert(isfinite3_safe(weight));
 
-  if (sc == NULL)
-    return NULL;
+  const float sample_weight = fabsf(average(weight));
 
-  float sample_weight = fabsf(average(weight));
-  sc->sample_weight = sample_weight;
-  return (sample_weight >= CLOSURE_WEIGHT_CUTOFF) ? sc : NULL;
+  /* Use comparison this way to help dealing with non-finite weight: if the average is not finite
+   * we will not allocate new closure. */
+  if (sample_weight >= CLOSURE_WEIGHT_CUTOFF) {
+    ShaderClosure *sc = closure_alloc(sd, size, CLOSURE_NONE_ID, weight);
+    if (sc == NULL) {
+      return NULL;
+    }
+
+    sc->sample_weight = sample_weight;
+
+    return sc;
+  }
+
+  return NULL;
 }
 
 #ifdef __OSL__
@@ -73,17 +83,27 @@ ccl_device_inline ShaderClosure *bsdf_alloc_osl(ShaderData *sd,
                                                 float3 weight,
                                                 void *data)
 {
-  ShaderClosure *sc = closure_alloc(sd, size, CLOSURE_NONE_ID, weight);
+  kernel_assert(isfinite3_safe(weight));
 
-  if (!sc)
-    return NULL;
+  const float sample_weight = fabsf(average(weight));
 
-  memcpy((void *)sc, data, size);
+  /* Use comparison this way to help dealing with non-finite weight: if the average is not finite
+   * we will not allocate new closure. */
+  if (sample_weight >= CLOSURE_WEIGHT_CUTOFF) {
+    ShaderClosure *sc = closure_alloc(sd, size, CLOSURE_NONE_ID, weight);
+    if (!sc) {
+      return NULL;
+    }
 
-  float sample_weight = fabsf(average(weight));
-  sc->weight = weight;
-  sc->sample_weight = sample_weight;
-  return (sample_weight >= CLOSURE_WEIGHT_CUTOFF) ? sc : NULL;
+    memcpy((void *)sc, data, size);
+
+    sc->weight = weight;
+    sc->sample_weight = sample_weight;
+
+    return sc;
+  }
+
+  return NULL;
 }
 #endif
 

intern/cycles/kernel/kernel_light.h

@@ -119,11 +119,11 @@ ccl_device_inline bool lamp_light_sample(
           klight->area.axisu[0], klight->area.axisu[1], klight->area.axisu[2]);
       float3 axisv = make_float3(
           klight->area.axisv[0], klight->area.axisv[1], klight->area.axisv[2]);
-      float3 D = make_float3(klight->area.dir[0], klight->area.dir[1], klight->area.dir[2]);
+      float3 Ng = make_float3(klight->area.dir[0], klight->area.dir[1], klight->area.dir[2]);
       float invarea = fabsf(klight->area.invarea);
       bool is_round = (klight->area.invarea < 0.0f);
 
-      if (dot(ls->P - P, D) > 0.0f) {
+      if (dot(ls->P - P, Ng) > 0.0f) {
         return false;
       }
 
@@ -136,19 +136,37 @@ ccl_device_inline bool lamp_light_sample(
       }
       else {
         inplane = ls->P;
-        ls->pdf = rect_light_sample(P, &ls->P, axisu, axisv, randu, randv, true);
+
+        float3 sample_axisu = axisu;
+        float3 sample_axisv = axisv;
+
+        if (klight->area.tan_spread > 0.0f) {
+          if (!light_spread_clamp_area_light(
+                  P, Ng, &ls->P, &sample_axisu, &sample_axisv, klight->area.tan_spread)) {
+            return false;
+          }
+        }
+
+        ls->pdf = rect_light_sample(P, &ls->P, sample_axisu, sample_axisv, randu, randv, true);
         inplane = ls->P - inplane;
       }
 
       ls->u = dot(inplane, axisu) * (1.0f / dot(axisu, axisu)) + 0.5f;
       ls->v = dot(inplane, axisv) * (1.0f / dot(axisv, axisv)) + 0.5f;
 
-      ls->Ng = D;
+      ls->Ng = Ng;
       ls->D = normalize_len(ls->P - P, &ls->t);
 
       ls->eval_fac = 0.25f * invarea;
+
+      if (klight->area.tan_spread > 0.0f) {
+        /* Area Light spread angle attenuation */
+        ls->eval_fac *= light_spread_attenuation(
+            ls->D, ls->Ng, klight->area.tan_spread, klight->area.normalize_spread);
+      }
+
       if (is_round) {
-        ls->pdf *= lamp_light_pdf(kg, D, -ls->D, ls->t);
+        ls->pdf *= lamp_light_pdf(kg, Ng, -ls->D, ls->t);
       }
     }
   }
@@ -283,9 +301,28 @@ ccl_device bool lamp_light_eval(
       ls->pdf = invarea * lamp_light_pdf(kg, Ng, -D, ls->t);
     }
     else {
-      ls->pdf = rect_light_sample(P, &light_P, axisu, axisv, 0, 0, false);
+      float3 sample_axisu = axisu;
+      float3 sample_axisv = axisv;
+
+      if (klight->area.tan_spread > 0.0f) {
+        if (!light_spread_clamp_area_light(
+                P, Ng, &light_P, &sample_axisu, &sample_axisv, klight->area.tan_spread)) {
+          return false;
+        }
+      }
+
+      ls->pdf = rect_light_sample(P, &light_P, sample_axisu, sample_axisv, 0, 0, false);
     }
     ls->eval_fac = 0.25f * invarea;
+
+    if (klight->area.tan_spread > 0.0f) {
+      /* Area Light spread angle attenuation */
+      ls->eval_fac *= light_spread_attenuation(
+          ls->D, ls->Ng, klight->area.tan_spread, klight->area.normalize_spread);
+      if (ls->eval_fac == 0.0f) {
+        return false;
+      }
+    }
   }
   else {
     return false;

intern/cycles/kernel/kernel_light_common.h

@@ -146,6 +146,70 @@ ccl_device float spot_light_attenuation(float3 dir, float spot_angle, float spot
   return attenuation;
 }
 
+ccl_device float light_spread_attenuation(const float3 D,
+                                          const float3 lightNg,
+                                          const float tan_spread,
+                                          const float normalize_spread)
+{
+  /* Model a soft-box grid, computing the ratio of light not hidden by the
+   * slats of the grid at a given angle. (see D10594). */
+  const float cos_a = -dot(D, lightNg);
+  const float sin_a = safe_sqrtf(1.0f - sqr(cos_a));
+  const float tan_a = sin_a / cos_a;
+  return max((1.0f - (tan_spread * tan_a)) * normalize_spread, 0.0f);
+}
+
+/* Compute subset of area light that actually has an influence on the shading point, to
+ * reduce noise with low spread. */
+ccl_device bool light_spread_clamp_area_light(const float3 P,
+                                              const float3 lightNg,
+                                              float3 *lightP,
+                                              float3 *axisu,
+                                              float3 *axisv,
+                                              const float tan_spread)
+{
+  /* Closest point in area light plane and distance to that plane. */
+  const float3 closest_P = P - dot(lightNg, P - *lightP) * lightNg;
+  const float t = len(closest_P - P);
+
+  /* Radius of circle on area light that actually affects the shading point. */
+  const float radius = t / tan_spread;
+
+  /* TODO: would be faster to store as normalized vector + length, also in rect_light_sample. */
+  float len_u, len_v;
+  const float3 u = normalize_len(*axisu, &len_u);
+  const float3 v = normalize_len(*axisv, &len_v);
+
+  /* Local uv coordinates of closest point. */
+  const float closest_u = dot(u, closest_P - *lightP);
+  const float closest_v = dot(v, closest_P - *lightP);
+
+  /* Compute rectangle encompassing the circle that affects the shading point,
+   * clamped to the bounds of the area light. */
+  const float min_u = max(closest_u - radius, -len_u * 0.5f);
+  const float max_u = min(closest_u + radius, len_u * 0.5f);
+  const float min_v = max(closest_v - radius, -len_v * 0.5f);
+  const float max_v = min(closest_v + radius, len_v * 0.5f);
+
+  /* Skip if rectangle is empty. */
+  if (min_u >= max_u || min_v >= max_v) {
+    return false;
+  }
+
+  /* Compute new area light center position and axes from rectangle in local
+   * uv coordinates. */
+  const float new_center_u = 0.5f * (min_u + max_u);
+  const float new_center_v = 0.5f * (min_v + max_v);
+  const float new_len_u = max_u - min_u;
+  const float new_len_v = max_v - min_v;
+
+  *lightP = *lightP + new_center_u * u + new_center_v * v;
+  *axisu = u * new_len_u;
+  *axisv = v * new_len_v;
+
+  return true;
+}
+
 ccl_device float lamp_light_pdf(KernelGlobals *kg, const float3 Ng, const float3 I, float t)
 {
   float cos_pi = dot(Ng, I);

intern/cycles/kernel/kernel_montecarlo.h

@@ -195,31 +195,108 @@ ccl_device float2 regular_polygon_sample(float corners, float rotation, float u,
 
 ccl_device float3 ensure_valid_reflection(float3 Ng, float3 I, float3 N)
 {
-  float3 R;
-  float NI = dot(N, I);
-  float NgR, threshold;
+  float3 R = 2 * dot(N, I) * N - I;
 
-  /* Check if the incident ray is coming from behind normal N. */
-  if (NI > 0) {
-    /* Normal reflection */
-    R = (2 * NI) * N - I;
-    NgR = dot(Ng, R);
+  /* Reflection rays may always be at least as shallow as the incoming ray. */
+  float threshold = min(0.9f * dot(Ng, I), 0.01f);
+  if (dot(Ng, R) >= threshold) {
+    return N;
+  }
 
-    /* Reflection rays may always be at least as shallow as the incoming ray. */
-    threshold = min(0.9f * dot(Ng, I), 0.01f);
-    if (NgR >= threshold) {
-      return N;
+  /* Form coordinate system with Ng as the Z axis and N inside the X-Z-plane.
+   * The X axis is found by normalizing the component of N that's orthogonal to Ng.
+   * The Y axis isn't actually needed.
+   */
+  float NdotNg = dot(N, Ng);
+  float3 X = normalize(N - NdotNg * Ng);
+
+  /* Keep math expressions. */
+  /* clang-format off */
+  /* Calculate N.z and N.x in the local coordinate system.
+   *
+   * The goal of this computation is to find a N' that is rotated towards Ng just enough
+   * to lift R' above the threshold (here called t), therefore dot(R', Ng) = t.
+   *
+   * According to the standard reflection equation,
+   * this means that we want dot(2*dot(N', I)*N' - I, Ng) = t.
+   *
+   * Since the Z axis of our local coordinate system is Ng, dot(x, Ng) is just x.z, so we get
+   * 2*dot(N', I)*N'.z - I.z = t.
+   *
+   * The rotation is simple to express in the coordinate system we formed -
+   * since N lies in the X-Z-plane, we know that N' will also lie in the X-Z-plane,
+   * so N'.y = 0 and therefore dot(N', I) = N'.x*I.x + N'.z*I.z .
+   *
+   * Furthermore, we want N' to be normalized, so N'.x = sqrt(1 - N'.z^2).
+   *
+   * With these simplifications,
+   * we get the final equation 2*(sqrt(1 - N'.z^2)*I.x + N'.z*I.z)*N'.z - I.z = t.
+   *
+   * The only unknown here is N'.z, so we can solve for that.
+   *
+   * The equation has four solutions in general:
+   *
+   * N'.z = +-sqrt(0.5*(+-sqrt(I.x^2*(I.x^2 + I.z^2 - t^2)) + t*I.z + I.x^2 + I.z^2)/(I.x^2 + I.z^2))
+   * We can simplify this expression a bit by grouping terms:
+   *
+   * a = I.x^2 + I.z^2
+   * b = sqrt(I.x^2 * (a - t^2))
+   * c = I.z*t + a
+   * N'.z = +-sqrt(0.5*(+-b + c)/a)
+   *
+   * Two solutions can immediately be discarded because they're negative so N' would lie in the
+   * lower hemisphere.
+   */
+  /* clang-format on */
+
+  float Ix = dot(I, X), Iz = dot(I, Ng);
+  float Ix2 = sqr(Ix), Iz2 = sqr(Iz);
+  float a = Ix2 + Iz2;
+
+  float b = safe_sqrtf(Ix2 * (a - sqr(threshold)));
+  float c = Iz * threshold + a;
+
+  /* Evaluate both solutions.
+   * In many cases one can be immediately discarded (if N'.z would be imaginary or larger than
+   * one), so check for that first. If no option is viable (might happen in extreme cases like N
+   * being in the wrong hemisphere), give up and return Ng. */
+  float fac = 0.5f / a;
+  float N1_z2 = fac * (b + c), N2_z2 = fac * (-b + c);
+  bool valid1 = (N1_z2 > 1e-5f) && (N1_z2 <= (1.0f + 1e-5f));
+  bool valid2 = (N2_z2 > 1e-5f) && (N2_z2 <= (1.0f + 1e-5f));
+
+  float2 N_new;
+  if (valid1 && valid2) {
+    /* If both are possible, do the expensive reflection-based check. */
+    float2 N1 = make_float2(safe_sqrtf(1.0f - N1_z2), safe_sqrtf(N1_z2));
+    float2 N2 = make_float2(safe_sqrtf(1.0f - N2_z2), safe_sqrtf(N2_z2));
+
+    float R1 = 2 * (N1.x * Ix + N1.y * Iz) * N1.y - Iz;
+    float R2 = 2 * (N2.x * Ix + N2.y * Iz) * N2.y - Iz;
+
+    valid1 = (R1 >= 1e-5f);
+    valid2 = (R2 >= 1e-5f);
+    if (valid1 && valid2) {
+      /* If both solutions are valid, return the one with the shallower reflection since it will be
+       * closer to the input (if the original reflection wasn't shallow, we would not be in this
+       * part of the function). */
+      N_new = (R1 < R2) ? N1 : N2;
+    }
+    else {
+      /* If only one reflection is valid (= positive), pick that one. */
+      N_new = (R1 > R2) ? N1 : N2;
     }
   }
+  else if (valid1 || valid2) {
+    /* Only one solution passes the N'.z criterium, so pick that one. */
+    float Nz2 = valid1 ? N1_z2 : N2_z2;
+    N_new = make_float2(safe_sqrtf(1.0f - Nz2), safe_sqrtf(Nz2));
+  }
   else {
-    /* Bad incident */
-    R = -I;
-    NgR = dot(Ng, R);
-    threshold = 0.01f;
+    return Ng;
   }
 
-  R = R + Ng * (threshold - NgR);            /* Lift the reflection above the threshold. */
-  return normalize(I * len(R) + R * len(I)); /* Find a bisector. */
+  return N_new.x * X + N_new.y * Ng;
 }
 
 CCL_NAMESPACE_END

intern/cycles/kernel/kernel_subsurface.h

@@ -25,8 +25,9 @@ CCL_NAMESPACE_BEGIN
 ccl_device_inline float3
 subsurface_scatter_eval(ShaderData *sd, const ShaderClosure *sc, float disk_r, float r, bool all)
 {
-  /* this is the veach one-sample model with balance heuristic, some pdf
-   * factors drop out when using balance heuristic weighting */
+  /* This is the Veach one-sample model with balance heuristic, some pdf
+   * factors drop out when using balance heuristic weighting. For branched
+   * path tracing (all) we sample all closure and don't use MIS. */
   float3 eval_sum = zero_float3();
   float pdf_sum = 0.0f;
   float sample_weight_inv = 0.0f;
@@ -65,6 +66,30 @@ subsurface_scatter_eval(ShaderData *sd, const ShaderClosure *sc, float disk_r, f
   return (pdf_sum > 0.0f) ? eval_sum / pdf_sum : zero_float3();
 }
 
+ccl_device_inline float3 subsurface_scatter_walk_eval(ShaderData *sd,
+                                                      const ShaderClosure *sc,
+                                                      float3 throughput,
+                                                      bool all)
+{
+  /* This is the Veach one-sample model with balance heuristic, some pdf
+   * factors drop out when using balance heuristic weighting. For branched
+   * path tracing (all) we sample all closure and don't use MIS. */
+  if (!all) {
+    float bssrdf_weight = 0.0f;
+    float weight = sc->sample_weight;
+
+    for (int i = 0; i < sd->num_closure; i++) {
+      sc = &sd->closure[i];
+
+      if (CLOSURE_IS_BSSRDF(sc->type)) {
+        bssrdf_weight += sc->sample_weight;
+      }
+    }
+    throughput *= bssrdf_weight / weight;
+  }
+  return throughput;
+}
+
 /* replace closures with a single diffuse bsdf closure after scatter step */
 ccl_device void subsurface_scatter_setup_diffuse_bsdf(
     KernelGlobals *kg, ShaderData *sd, ClosureType type, float roughness, float3 weight, float3 N)
@@ -437,7 +462,8 @@ ccl_device_noinline
                            ccl_addr_space PathState *state,
                            const ShaderClosure *sc,
                            const float bssrdf_u,
-                           const float bssrdf_v)
+                           const float bssrdf_v,
+                           bool all)
 {
   /* Sample diffuse surface scatter into the object. */
   float3 D;
@@ -605,6 +631,13 @@ ccl_device_noinline
     if (hit) {
       t = ray->t;
     }
+    else if (bounce == 0) {
+      /* Restore original position if nothing was hit after the first bounce,
+       * without the ray_offset() that was added to avoid self-intersection.
+       * Otherwise if that offset is relatively large compared to the scattering
+       * radius, we never go back up high enough to exit the surface. */
+      ray->P = sd->P;
+    }
 
     /* Advance to new scatter location. */
     ray->P += t * ray->D;
@@ -662,7 +695,7 @@ ccl_device_noinline
   /* TODO: gain back performance lost from merging with disk BSSRDF. We
    * only need to return on hit so this indirect ray push/pop overhead
    * is not actually needed, but it does keep the code simpler. */
-  ss_isect->weight[0] = throughput;
+  ss_isect->weight[0] = subsurface_scatter_walk_eval(sd, sc, throughput, all);
 #ifdef __SPLIT_KERNEL__
   ss_isect->ray = *ray;
 #endif
@@ -684,7 +717,7 @@ ccl_device_inline int subsurface_scatter_multi_intersect(KernelGlobals *kg,
     return subsurface_scatter_disk(kg, ss_isect, sd, sc, lcg_state, bssrdf_u, bssrdf_v, all);
   }
   else {
-    return subsurface_random_walk(kg, ss_isect, sd, state, sc, bssrdf_u, bssrdf_v);
+    return subsurface_random_walk(kg, ss_isect, sd, state, sc, bssrdf_u, bssrdf_v, all);
   }
 }
 

intern/cycles/kernel/kernel_types.h

@@ -99,27 +99,23 @@ CCL_NAMESPACE_BEGIN
 #define __AO__
 #define __PASSES__
 #define __HAIR__
-
-/* Without these we get an AO render, used by OpenCL preview kernel. */
-#ifndef __KERNEL_AO_PREVIEW__
-#  define __SVM__
-#  define __EMISSION__
-#  define __HOLDOUT__
-#  define __MULTI_CLOSURE__
-#  define __TRANSPARENT_SHADOWS__
-#  define __BACKGROUND_MIS__
-#  define __LAMP_MIS__
-#  define __CAMERA_MOTION__
-#  define __OBJECT_MOTION__
-#  define __BAKING__
-#  define __PRINCIPLED__
-#  define __SUBSURFACE__
-#  define __VOLUME__
-#  define __VOLUME_SCATTER__
-#  define __CMJ__
-#  define __SHADOW_RECORD_ALL__
-#  define __BRANCHED_PATH__
-#endif
+#define __SVM__
+#define __EMISSION__
+#define __HOLDOUT__
+#define __MULTI_CLOSURE__
+#define __TRANSPARENT_SHADOWS__
+#define __BACKGROUND_MIS__
+#define __LAMP_MIS__
+#define __CAMERA_MOTION__
+#define __OBJECT_MOTION__
+#define __BAKING__
+#define __PRINCIPLED__
+#define __SUBSURFACE__
+#define __VOLUME__
+#define __VOLUME_SCATTER__
+#define __CMJ__
+#define __SHADOW_RECORD_ALL__
+#define __BRANCHED_PATH__
 
 /* Device specific features */
 #ifdef __KERNEL_CPU__
@@ -1501,9 +1497,9 @@ typedef struct KernelAreaLight {
   float axisu[3];
   float invarea;
   float axisv[3];
-  float pad1;
+  float tan_spread;
   float dir[3];
-  float pad2;
+  float normalize_spread;
 } KernelAreaLight;
 
 typedef struct KernelDistantLight {

intern/cycles/kernel/shaders/stdcycles.h

@@ -84,30 +84,67 @@ closure color principled_hair(normal N,
 closure color henyey_greenstein(float g) BUILTIN;
 closure color absorption() BUILTIN;
 
-normal ensure_valid_reflection(normal Ng, normal I, normal N)
+normal ensure_valid_reflection(normal Ng, vector I, normal N)
 {
   /* The implementation here mirrors the one in kernel_montecarlo.h,
    * check there for an explanation of the algorithm. */
-  vector R;
-  float NI = dot(N, I);
-  float NgR, threshold;
 
-  if (NI > 0) {
-    R = (2 * NI) * N - I;
-    NgR = dot(Ng, R);
-    threshold = min(0.9 * dot(Ng, I), 0.01);
-    if (NgR >= threshold) {
-      return N;
+  float sqr(float x)
+  {
+    return x * x;
+  }
+
+  vector R = 2 * dot(N, I) * N - I;
+
+  float threshold = min(0.9 * dot(Ng, I), 0.01);
+  if (dot(Ng, R) >= threshold) {
+    return N;
+  }
+
+  float NdotNg = dot(N, Ng);
+  vector X = normalize(N - NdotNg * Ng);
+
+  float Ix = dot(I, X), Iz = dot(I, Ng);
+  float Ix2 = sqr(Ix), Iz2 = sqr(Iz);
+  float a = Ix2 + Iz2;
+
+  float b = sqrt(Ix2 * (a - sqr(threshold)));
+  float c = Iz * threshold + a;
+
+  float fac = 0.5 / a;
+  float N1_z2 = fac * (b + c), N2_z2 = fac * (-b + c);
+  int valid1 = (N1_z2 > 1e-5) && (N1_z2 <= (1.0 + 1e-5));
+  int valid2 = (N2_z2 > 1e-5) && (N2_z2 <= (1.0 + 1e-5));
+
+  float N_new_x, N_new_z;
+  if (valid1 && valid2) {
+    float N1_x = sqrt(1.0 - N1_z2), N1_z = sqrt(N1_z2);
+    float N2_x = sqrt(1.0 - N2_z2), N2_z = sqrt(N2_z2);
+
+    float R1 = 2 * (N1_x * Ix + N1_z * Iz) * N1_z - Iz;
+    float R2 = 2 * (N2_x * Ix + N2_z * Iz) * N2_z - Iz;
+
+    valid1 = (R1 >= 1e-5);
+    valid2 = (R2 >= 1e-5);
+    if (valid1 && valid2) {
+      N_new_x = (R1 < R2) ? N1_x : N2_x;
+      N_new_z = (R1 < R2) ? N1_z : N2_z;
+    }
+    else {
+      N_new_x = (R1 > R2) ? N1_x : N2_x;
+      N_new_z = (R1 > R2) ? N1_z : N2_z;
     }
   }
+  else if (valid1 || valid2) {
+    float Nz2 = valid1 ? N1_z2 : N2_z2;
+    N_new_x = sqrt(1.0 - Nz2);
+    N_new_z = sqrt(Nz2);
+  }
   else {
-    R = -I;
-    NgR = dot(Ng, R);
-    threshold = 0.01;
+    return Ng;
   }
 
-  R = R + Ng * (threshold - NgR);
-  return normalize(I * length(R) + R * length(I));
+  return N_new_x * X + N_new_z * Ng;
 }
 
 #endif /* CCL_STDOSL_H */

intern/cycles/kernel/svm/svm_tex_coord.h

@@ -370,10 +370,13 @@ ccl_device void svm_node_tangent(KernelGlobals *kg, ShaderData *sd, float *stack
 
   if (direction_type == NODE_TANGENT_UVMAP) {
     /* UV map */
-    if (desc.offset == ATTR_STD_NOT_FOUND)
-      tangent = make_float3(0.0f, 0.0f, 0.0f);
-    else
+    if (desc.offset == ATTR_STD_NOT_FOUND) {
+      stack_store_float3(stack, tangent_offset, zero_float3());
+      return;
+    }
+    else {
       tangent = attribute_value;
+    }
   }
   else {
     /* radial */

intern/cycles/render/alembic.cpp

@@ -1407,6 +1407,15 @@ void AlembicProcedural::generate(Scene *scene, Progress &progress)
       need_data_updates = true;
     }
 
+    /* Check if the shaders were modified. */
+    if (object->used_shaders_is_modified() && object->get_object() &&
+        object->get_object()->get_geometry()) {
+      Geometry *geometry = object->get_object()->get_geometry();
+      array<Node *> used_shaders = object->get_used_shaders();
+      geometry->set_used_shaders(used_shaders);
+      need_shader_updates = true;
+    }
+
     /* Check for changes in shaders (e.g. newly requested attributes). */
     foreach (Node *shader_node, object->get_used_shaders()) {
       Shader *shader = static_cast<Shader *>(shader_node);
@@ -1586,6 +1595,11 @@ void AlembicProcedural::read_mesh(AlembicObject *abc_object, Abc::chrono_t frame
 
   Mesh *mesh = static_cast<Mesh *>(object->get_geometry());
 
+  /* Make sure shader ids are also updated. */
+  if (mesh->used_shaders_is_modified()) {
+    mesh->tag_shader_modified();
+  }
+
   cached_data.vertices.copy_to_socket(frame_time, mesh, mesh->get_verts_socket());
 
   cached_data.shader.copy_to_socket(frame_time, mesh, mesh->get_shader_socket());
@@ -1653,6 +1667,11 @@ void AlembicProcedural::read_subd(AlembicObject *abc_object, Abc::chrono_t frame
 
   Mesh *mesh = static_cast<Mesh *>(object->get_geometry());
 
+  /* Make sure shader ids are also updated. */
+  if (mesh->used_shaders_is_modified()) {
+    mesh->tag_shader_modified();
+  }
+
   /* Cycles overwrites the original triangles when computing displacement, so we always have to
    * repass the data if something is animated (vertices most likely) to avoid buffer overflows. */
   if (!cached_data.is_constant()) {
@@ -1743,6 +1762,11 @@ void AlembicProcedural::read_curves(AlembicObject *abc_object, Abc::chrono_t fra
 
   Hair *hair = static_cast<Hair *>(object->get_geometry());
 
+  /* Make sure shader ids are also updated. */
+  if (hair->used_shaders_is_modified()) {
+    hair->tag_curve_shader_modified();
+  }
+
   cached_data.curve_keys.copy_to_socket(frame_time, hair, hair->get_curve_keys_socket());
 
   cached_data.curve_radius.copy_to_socket(frame_time, hair, hair->get_curve_radius_socket());

intern/cycles/render/alembic.h

@@ -239,6 +239,7 @@ template<typename T> class DataStore {
   {
     invalidate_last_loaded_time();
     data.clear();
+    index_data_map.clear();
   }
 
   void invalidate_last_loaded_time()
@@ -480,16 +481,23 @@ class AlembicProcedural : public Procedural {
    * invocation, and updates the data on subsequent invocations if the frame changed. */
   void generate(Scene *scene, Progress &progress);
 
-  /* Add an object to our list of objects, and tag the socket as modified. */
-  void add_object(AlembicObject *object);
-
   /* Tag for an update only if something was modified. */
   void tag_update(Scene *scene);
 
-  /* Returns a pointer to an existing or a newly created AlembicObject for the given path. */
+  /* This should be called by scene exporters to request the rendering of an object located
+   * in the Alembic archive at the given path.
+   *
+   * Since we lazily load object, the function does not validate the existence of the object
+   * in the archive. If no objects with such path if found in the archive during the next call
+   * to `generate`, it will be ignored.
+   *
+   * Returns a pointer to an existing or a newly created AlembicObject for the given path. */
   AlembicObject *get_or_create_object(const ustring &path);
 
  private:
+  /* Add an object to our list of objects, and tag the socket as modified. */
+  void add_object(AlembicObject *object);
+
   /* Load the data for all the objects whose data has not yet been loaded. */
   void load_objects(Progress &progress);
 

intern/cycles/render/attribute.cpp

@@ -689,6 +689,9 @@ void AttributeSet::update(AttributeSet &&new_attributes)
 
     it++;
   }
+
+  /* If all attributes were replaced, transform is no longer applied. */
+  geometry->transform_applied = false;
 }
 
 void AttributeSet::clear_modified()

intern/cycles/render/geometry.cpp

@@ -46,6 +46,12 @@ CCL_NAMESPACE_BEGIN
 
 /* Geometry */
 
+PackFlags operator|=(PackFlags &pack_flags, uint32_t value)
+{
+  pack_flags = (PackFlags)((uint32_t)pack_flags | value);
+  return pack_flags;
+}
+
 NODE_ABSTRACT_DEFINE(Geometry)
 {
   NodeType *type = NodeType::add("geometry_base", NULL);
@@ -1235,7 +1241,16 @@ void GeometryManager::device_update_bvh(Device *device,
 
   const bool can_refit = scene->bvh != nullptr &&
                          (bparams.bvh_layout == BVHLayout::BVH_LAYOUT_OPTIX);
-  const bool pack_all = scene->bvh == nullptr;
+
+  PackFlags pack_flags = PackFlags::PACK_NONE;
+
+  if (scene->bvh == nullptr) {
+    pack_flags |= PackFlags::PACK_ALL;
+  }
+
+  if (dscene->prim_visibility.is_modified()) {
+    pack_flags |= PackFlags::PACK_VISIBILITY;
+  }
 
   BVH *bvh = scene->bvh;
   if (!scene->bvh) {
@@ -1273,10 +1288,14 @@ void GeometryManager::device_update_bvh(Device *device,
 
     pack.root_index = -1;
 
-    if (!pack_all) {
+    if (pack_flags != PackFlags::PACK_ALL) {
       /* if we do not need to recreate the BVH, then only the vertices are updated, so we can
        * safely retake the memory */
       dscene->prim_tri_verts.give_data(pack.prim_tri_verts);
+
+      if ((pack_flags & PackFlags::PACK_VISIBILITY) != 0) {
+        dscene->prim_visibility.give_data(pack.prim_visibility);
+      }
     }
     else {
       /* It is not strictly necessary to skip those resizes we if do not have to repack, as the OS
@@ -1305,13 +1324,21 @@ void GeometryManager::device_update_bvh(Device *device,
     // Iterate over scene mesh list instead of objects, since 'optix_prim_offset' was calculated
     // based on that list, which may be ordered differently from the object list.
     foreach (Geometry *geom, scene->geometry) {
-      if (!pack_all && !geom->is_modified()) {
+      /* Make a copy of the pack_flags so the current geometry's flags do not pollute the others'.
+       */
+      PackFlags geom_pack_flags = pack_flags;
+
+      if (geom->is_modified()) {
+        geom_pack_flags |= PackFlags::PACK_VERTICES;
+      }
+
+      if (geom_pack_flags == PACK_NONE) {
         continue;
       }
 
       const pair<int, uint> &info = geometry_to_object_info[geom];
       pool.push(function_bind(
-          &Geometry::pack_primitives, geom, &pack, info.first, info.second, pack_all));
+          &Geometry::pack_primitives, geom, &pack, info.first, info.second, geom_pack_flags));
     }
     pool.wait_work();
   }
@@ -1346,7 +1373,7 @@ void GeometryManager::device_update_bvh(Device *device,
     dscene->prim_type.steal_data(pack.prim_type);
     dscene->prim_type.copy_to_device();
   }
-  if (pack.prim_visibility.size() && (dscene->prim_visibility.need_realloc() || has_bvh2_layout)) {
+  if (pack.prim_visibility.size() && (dscene->prim_visibility.is_modified() || has_bvh2_layout)) {
     dscene->prim_visibility.steal_data(pack.prim_visibility);
     dscene->prim_visibility.copy_to_device();
   }
@@ -1364,7 +1391,6 @@ void GeometryManager::device_update_bvh(Device *device,
   }
 
   dscene->data.bvh.root = pack.root_index;
-  dscene->data.bvh.bvh_layout = bparams.bvh_layout;
   dscene->data.bvh.use_bvh_steps = (scene->params.num_bvh_time_steps != 0);
   dscene->data.bvh.curve_subdivisions = scene->params.curve_subdivisions();
   /* The scene handle is set in 'CPUDevice::const_copy_to' and 'OptiXDevice::const_copy_to' */
@@ -1595,6 +1621,10 @@ void GeometryManager::device_update_preprocess(Device *device, Scene *scene, Pro
     }
   }
 
+  if ((update_flags & VISIBILITY_MODIFIED) != 0) {
+    dscene->prim_visibility.tag_modified();
+  }
+
   if (device_update_flags & ATTR_FLOAT_NEEDS_REALLOC) {
     dscene->attributes_map.tag_realloc();
     dscene->attributes_float.tag_realloc();
@@ -1921,7 +1951,8 @@ void GeometryManager::device_update(Device *device,
    * Also update the BVH if the transformations change, we cannot rely on tagging the Geometry
    * as modified in this case, as we may accumulate displacement if the vertices do not also
    * change. */
-  bool need_update_scene_bvh = (scene->bvh == nullptr || (update_flags & TRANSFORM_MODIFIED) != 0);
+  bool need_update_scene_bvh = (scene->bvh == nullptr ||
+                                (update_flags & (TRANSFORM_MODIFIED | VISIBILITY_MODIFIED)) != 0);
   {
     scoped_callback_timer timer([scene](double time) {
       if (scene->update_stats) {
@@ -1984,6 +2015,11 @@ void GeometryManager::device_update(Device *device,
     }
   }
 
+  /* Always set BVH layout again after displacement where it was set to none,
+   * to avoid ray-tracing at that stage. */
+  dscene->data.bvh.bvh_layout = BVHParams::best_bvh_layout(scene->params.bvh_layout,
+                                                           device->get_bvh_layout_mask());
+
   {
     scoped_callback_timer timer([scene](double time) {
       if (scene->update_stats) {

intern/cycles/render/geometry.h

@@ -43,6 +43,24 @@ class Shader;
 class Volume;
 struct PackedBVH;
 
+/* Flags used to determine which geometry data need to be packed. */
+enum PackFlags : uint32_t {
+  PACK_NONE = 0u,
+
+  /* Pack the geometry information (e.g. triangle or curve keys indices). */
+  PACK_GEOMETRY = (1u << 0),
+
+  /* Pack the vertice, for Meshes and Volumes' bouding meshes. */
+  PACK_VERTICES = (1u << 1),
+
+  /* Pack the visibility flags for each triangle or curve. */
+  PACK_VISIBILITY = (1u << 2),
+
+  PACK_ALL = (PACK_GEOMETRY | PACK_VERTICES | PACK_VISIBILITY),
+};
+
+PackFlags operator|=(PackFlags &pack_flags, uint32_t value);
+
 /* Geometry
  *
  * Base class for geometric types like Mesh and Hair. */
@@ -126,7 +144,10 @@ class Geometry : public Node {
                    int n,
                    int total);
 
-  virtual void pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all) = 0;
+  virtual void pack_primitives(PackedBVH *pack,
+                               int object,
+                               uint visibility,
+                               PackFlags pack_flags) = 0;
 
   /* Check whether the geometry should have own BVH built separately. Briefly,
    * own BVH is needed for geometry, if:
@@ -191,6 +212,8 @@ class GeometryManager {
 
     TRANSFORM_MODIFIED = (1 << 10),
 
+    VISIBILITY_MODIFIED = (1 << 11),
+
     /* tag everything in the manager for an update */
     UPDATE_ALL = ~0u,
 

intern/cycles/render/hair.cpp

@@ -494,38 +494,47 @@ void Hair::pack_curves(Scene *scene,
   }
 }
 
-void Hair::pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all)
+void Hair::pack_primitives(PackedBVH *pack, int object, uint visibility, PackFlags pack_flags)
 {
   if (curve_first_key.empty())
     return;
 
-  /* If the BVH does not have to be recreated, we can bail out. */
-  if (!pack_all) {
-    return;
+  /* Separate loop as other arrays are not initialized if their packing is not required. */
+  if ((pack_flags & PACK_VISIBILITY) != 0) {
+    unsigned int *prim_visibility = &pack->prim_visibility[optix_prim_offset];
+
+    size_t index = 0;
+    for (size_t j = 0; j < num_curves(); ++j) {
+      Curve curve = get_curve(j);
+      for (size_t k = 0; k < curve.num_segments(); ++k, ++index) {
+        prim_visibility[index] = visibility;
+      }
+    }
   }
 
-  unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
-  int *prim_type = &pack->prim_type[optix_prim_offset];
-  unsigned int *prim_visibility = &pack->prim_visibility[optix_prim_offset];
-  int *prim_index = &pack->prim_index[optix_prim_offset];
-  int *prim_object = &pack->prim_object[optix_prim_offset];
-  // 'pack->prim_time' is unused by Embree and OptiX
+  if ((pack_flags & PACK_GEOMETRY) != 0) {
+    unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
+    int *prim_type = &pack->prim_type[optix_prim_offset];
+    int *prim_index = &pack->prim_index[optix_prim_offset];
+    int *prim_object = &pack->prim_object[optix_prim_offset];
+    // 'pack->prim_time' is unused by Embree and OptiX
 
-  uint type = has_motion_blur() ?
-                  ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_MOTION_CURVE_RIBBON :
-                                                   PRIMITIVE_MOTION_CURVE_THICK) :
-                  ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_CURVE_RIBBON : PRIMITIVE_CURVE_THICK);
+    uint type = has_motion_blur() ?
+                    ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_MOTION_CURVE_RIBBON :
+                                                     PRIMITIVE_MOTION_CURVE_THICK) :
+                    ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_CURVE_RIBBON :
+                                                     PRIMITIVE_CURVE_THICK);
 
-  size_t index = 0;
-  for (size_t j = 0; j < num_curves(); ++j) {
-    Curve curve = get_curve(j);
-    for (size_t k = 0; k < curve.num_segments(); ++k, ++index) {
-      prim_tri_index[index] = -1;
-      prim_type[index] = PRIMITIVE_PACK_SEGMENT(type, k);
-      prim_visibility[index] = visibility;
-      // Each curve segment points back to its curve index
-      prim_index[index] = j + prim_offset;
-      prim_object[index] = object;
+    size_t index = 0;
+    for (size_t j = 0; j < num_curves(); ++j) {
+      Curve curve = get_curve(j);
+      for (size_t k = 0; k < curve.num_segments(); ++k, ++index) {
+        prim_tri_index[index] = -1;
+        prim_type[index] = PRIMITIVE_PACK_SEGMENT(type, k);
+        // Each curve segment points back to its curve index
+        prim_index[index] = j + prim_offset;
+        prim_object[index] = object;
+      }
     }
   }
 }

intern/cycles/render/hair.h

@@ -146,7 +146,10 @@ class Hair : public Geometry {
   /* BVH */
   void pack_curves(Scene *scene, float4 *curve_key_co, float4 *curve_data, size_t curvekey_offset);
 
-  void pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all) override;
+  void pack_primitives(PackedBVH *pack,
+                       int object,
+                       uint visibility,
+                       PackFlags pack_flags) override;
 };
 
 CCL_NAMESPACE_END

intern/cycles/render/image_vdb.cpp

@@ -179,9 +179,13 @@ bool VDBImageLoader::load_metadata(const ImageDeviceFeatures &features, ImageMet
   metadata.transform_3d = transform_inverse(index_to_object * texture_to_index);
   metadata.use_transform_3d = true;
 
+#  ifndef WITH_NANOVDB
+  (void)features;
+#  endif
   return true;
 #else
   (void)metadata;
+  (void)features;
   return false;
 #endif
 }

intern/cycles/render/light.cpp

@@ -129,6 +129,7 @@ NODE_DEFINE(Light)
   SOCKET_VECTOR(axisv, "Axis V", zero_float3());
   SOCKET_FLOAT(sizev, "Size V", 1.0f);
   SOCKET_BOOLEAN(round, "Round", false);
+  SOCKET_FLOAT(spread, "Spread", M_PI_F);
 
   SOCKET_INT(map_resolution, "Map Resolution", 0);
 
@@ -858,6 +859,15 @@ void LightManager::device_update_points(Device *, DeviceScene *dscene, Scene *sc
       float invarea = (area != 0.0f) ? 1.0f / area : 1.0f;
       float3 dir = light->dir;
 
+      /* Convert from spread angle 0..180 to 90..0, clamping to a minimum
+       * angle to avoid excessive noise. */
+      const float min_spread_angle = 1.0f * M_PI_F / 180.0f;
+      const float spread_angle = 0.5f * (M_PI_F - max(light->spread, min_spread_angle));
+      /* Normalization computed using:
+       * integrate cos(x) * (1 - tan(x) * tan(a)) * sin(x) from x = 0 to pi/2 - a. */
+      const float tan_spread = tanf(spread_angle);
+      const float normalize_spread = 2.0f / (2.0f + (2.0f * spread_angle - M_PI_F) * tan_spread);
+
       dir = safe_normalize(dir);
 
       if (light->use_mis && area != 0.0f)
@@ -877,6 +887,8 @@ void LightManager::device_update_points(Device *, DeviceScene *dscene, Scene *sc
       klights[light_index].area.dir[0] = dir.x;
       klights[light_index].area.dir[1] = dir.y;
       klights[light_index].area.dir[2] = dir.z;
+      klights[light_index].area.tan_spread = tan_spread;
+      klights[light_index].area.normalize_spread = normalize_spread;
     }
     else if (light->light_type == LIGHT_SPOT) {
       shader_id &= ~SHADER_AREA_LIGHT;

intern/cycles/render/light.h

@@ -58,6 +58,7 @@ class Light : public Node {
   NODE_SOCKET_API(float3, axisv)
   NODE_SOCKET_API(float, sizev)
   NODE_SOCKET_API(bool, round)
+  NODE_SOCKET_API(float, spread)
 
   NODE_SOCKET_API(Transform, tfm)
 

intern/cycles/render/mesh.cpp

@@ -805,7 +805,7 @@ void Mesh::pack_patches(uint *patch_data, uint vert_offset, uint face_offset, ui
   }
 }
 
-void Mesh::pack_primitives(ccl::PackedBVH *pack, int object, uint visibility, bool pack_all)
+void Mesh::pack_primitives(ccl::PackedBVH *pack, int object, uint visibility, PackFlags pack_flags)
 {
   if (triangles.empty())
     return;
@@ -819,28 +819,38 @@ void Mesh::pack_primitives(ccl::PackedBVH *pack, int object, uint visibility, bo
 
   uint type = has_motion_blur() ? PRIMITIVE_MOTION_TRIANGLE : PRIMITIVE_TRIANGLE;
 
-  if (pack_all) {
-    /* Use optix_prim_offset for indexing as those arrays also contain data for Hair geometries. */
-    unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
-    int *prim_type = &pack->prim_type[optix_prim_offset];
+  /* Separate loop as other arrays are not initialized if their packing is not required. */
+  if ((pack_flags & PackFlags::PACK_VISIBILITY) != 0) {
     unsigned int *prim_visibility = &pack->prim_visibility[optix_prim_offset];
-    int *prim_index = &pack->prim_index[optix_prim_offset];
-    int *prim_object = &pack->prim_object[optix_prim_offset];
-
     for (size_t k = 0; k < num_prims; ++k) {
-      prim_tri_index[k] = (prim_offset + k) * 3;
-      prim_type[k] = type;
-      prim_index[k] = prim_offset + k;
-      prim_object[k] = object;
       prim_visibility[k] = visibility;
     }
   }
 
-  for (size_t k = 0; k < num_prims; ++k) {
-    const Mesh::Triangle t = get_triangle(k);
-    prim_tri_verts[k * 3] = float3_to_float4(verts[t.v[0]]);
-    prim_tri_verts[k * 3 + 1] = float3_to_float4(verts[t.v[1]]);
-    prim_tri_verts[k * 3 + 2] = float3_to_float4(verts[t.v[2]]);
+  if ((pack_flags & PackFlags::PACK_GEOMETRY) != 0) {
+    /* Use optix_prim_offset for indexing as those arrays also contain data for Hair geometries. */
+    unsigned int *prim_tri_index = &pack->prim_tri_index[optix_prim_offset];
+    int *prim_type = &pack->prim_type[optix_prim_offset];
+    int *prim_index = &pack->prim_index[optix_prim_offset];
+    int *prim_object = &pack->prim_object[optix_prim_offset];
+
+    for (size_t k = 0; k < num_prims; ++k) {
+      if ((pack_flags & PackFlags::PACK_GEOMETRY) != 0) {
+        prim_tri_index[k] = (prim_offset + k) * 3;
+        prim_type[k] = type;
+        prim_index[k] = prim_offset + k;
+        prim_object[k] = object;
+      }
+    }
+  }
+
+  if ((pack_flags & PackFlags::PACK_VERTICES) != 0) {
+    for (size_t k = 0; k < num_prims; ++k) {
+      const Mesh::Triangle t = get_triangle(k);
+      prim_tri_verts[k * 3] = float3_to_float4(verts[t.v[0]]);
+      prim_tri_verts[k * 3 + 1] = float3_to_float4(verts[t.v[1]]);
+      prim_tri_verts[k * 3 + 2] = float3_to_float4(verts[t.v[2]]);
+    }
   }
 }
 

intern/cycles/render/mesh.h

@@ -232,7 +232,10 @@ class Mesh : public Geometry {
                   size_t tri_offset);
   void pack_patches(uint *patch_data, uint vert_offset, uint face_offset, uint corner_offset);
 
-  void pack_primitives(PackedBVH *pack, int object, uint visibility, bool pack_all) override;
+  void pack_primitives(PackedBVH *pack,
+                       int object,
+                       uint visibility,
+                       PackFlags pack_flags) override;
 
   void tessellate(DiagSplit *split);
 

intern/cycles/render/nodes.h

@@ -1600,11 +1600,23 @@ class SetNormalNode : public ShaderNode {
   NODE_SOCKET_API(float3, direction)
 };
 
-class OSLNode : public ShaderNode {
+class OSLNode final : public ShaderNode {
  public:
   static OSLNode *create(ShaderGraph *graph, size_t num_inputs, const OSLNode *from = NULL);
   ~OSLNode();
 
+  static void operator delete(void *ptr)
+  {
+    /* Override delete operator to silence new-delete-type-mismatch ASAN warnings
+     * regarding size mismatch in the destructor. This is intentional as we allocate
+     * extra space at the end of the node. */
+    ::operator delete(ptr);
+  }
+  static void operator delete(void *, void *)
+  {
+    /* Deliberately empty placement delete operator, to avoid MSVC warning C4291. */
+  }
+
   ShaderNode *clone(ShaderGraph *graph) const;
 
   char *input_default_value();

intern/cycles/render/object.cpp

@@ -220,6 +220,10 @@ void Object::tag_update(Scene *scene)
       flag |= ObjectManager::TRANSFORM_MODIFIED;
     }
 
+    if (visibility_is_modified()) {
+      flag |= ObjectManager::VISIBILITY_MODIFIED;
+    }
+
     foreach (Node *node, geometry->get_used_shaders()) {
       Shader *shader = static_cast<Shader *>(node);
       if (shader->get_use_mis() && shader->has_surface_emission)
@@ -914,6 +918,10 @@ void ObjectManager::tag_update(Scene *scene, uint32_t flag)
       geometry_flag |= GeometryManager::TRANSFORM_MODIFIED;
     }
 
+    if ((flag & VISIBILITY_MODIFIED) != 0) {
+      geometry_flag |= GeometryManager::VISIBILITY_MODIFIED;
+    }
+
     scene->geometry_manager->tag_update(scene, geometry_flag);
   }
 

intern/cycles/render/object.h

@@ -134,6 +134,7 @@ class ObjectManager {
     OBJECT_MODIFIED = (1 << 5),
     HOLDOUT_MODIFIED = (1 << 6),
     TRANSFORM_MODIFIED = (1 << 7),
+    VISIBILITY_MODIFIED = (1 << 8),
 
     /* tag everything in the manager for an update */
     UPDATE_ALL = ~0u,

intern/cycles/render/scene.cpp

@@ -179,7 +179,7 @@ void Scene::free_memory(bool final)
 
     bake_manager->device_free(device, &dscene);
 
-    if (!params.persistent_data || final)
+    if (final)
       image_manager->device_free(device);
     else
       image_manager->device_free_builtin(device);
@@ -526,9 +526,6 @@ bool Scene::update(Progress &progress, bool &kernel_switch_needed)
     DeviceKernelStatus kernel_switch_status = device->get_active_kernel_switch_state();
     kernel_switch_needed = kernel_switch_status == DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE ||
                            kernel_switch_status == DEVICE_KERNEL_FEATURE_KERNEL_INVALID;
-    if (kernel_switch_status == DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL) {
-      progress.set_kernel_status("Compiling render kernels");
-    }
     if (new_kernels_needed || kernel_switch_needed) {
       progress.set_kernel_status("Compiling render kernels");
       device->wait_for_availability(loaded_kernel_features);
@@ -566,9 +563,6 @@ bool Scene::load_kernels(Progress &progress, bool lock_scene)
       return false;
     }
 
-    progress.add_skip_time(timer, false);
-    VLOG(1) << "Total time spent loading kernels: " << time_dt() - timer.get_start();
-
     kernels_loaded = true;
     loaded_kernel_features = requested_features;
     return true;

intern/cycles/render/scene.h

@@ -178,7 +178,6 @@ class SceneParams {
   int num_bvh_time_steps;
   int hair_subdivisions;
   CurveShapeType hair_shape;
-  bool persistent_data;
   int texture_limit;
 
   bool background;
@@ -193,7 +192,6 @@ class SceneParams {
     num_bvh_time_steps = 0;
     hair_subdivisions = 3;
     hair_shape = CURVE_RIBBON;
-    persistent_data = false;
     texture_limit = 0;
     background = true;
   }
@@ -206,7 +204,7 @@ class SceneParams {
              use_bvh_unaligned_nodes == params.use_bvh_unaligned_nodes &&
              num_bvh_time_steps == params.num_bvh_time_steps &&
              hair_subdivisions == params.hair_subdivisions && hair_shape == params.hair_shape &&
-             persistent_data == params.persistent_data && texture_limit == params.texture_limit);
+             texture_limit == params.texture_limit);
   }
 
   int curve_subdivisions()

intern/cycles/render/session.cpp

@@ -243,11 +243,6 @@ void Session::run_gpu()
       }
     }
 
-    /* Don't go in pause mode when image was rendered with preview kernels
-     * When feature kernels become available the session will be reset. */
-    else if (no_tiles && kernel_state == DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL) {
-      time_sleep(0.1);
-    }
     else if (no_tiles && kernel_state == DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE) {
       reset_gpu(tile_manager.params, params.samples);
     }
@@ -762,11 +757,6 @@ void Session::run_cpu()
       }
     }
 
-    /* Don't go in pause mode when preview kernels are used
-     * When feature kernels become available the session will be reset. */
-    else if (no_tiles && kernel_state == DEVICE_KERNEL_WAITING_FOR_FEATURE_KERNEL) {
-      time_sleep(0.1);
-    }
     else if (no_tiles && kernel_state == DEVICE_KERNEL_FEATURE_KERNEL_AVAILABLE) {
       reset_cpu(tile_manager.params, params.samples);
     }

intern/cycles/test/util_avxf_avx2_test.cpp

@@ -18,6 +18,7 @@
 
 #define TEST_CATEGORY_NAME util_avx2
 
-#if defined(i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#if (defined(i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)) && \
+    defined(__AVX2__)
 #  include "util_avxf_test.h"
 #endif

intern/cycles/test/util_avxf_avx_test.cpp

@@ -18,6 +18,7 @@
 
 #define TEST_CATEGORY_NAME util_avx
 
-#if defined(i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#if (defined(i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)) && \
+    defined(__AVX__)
 #  include "util_avxf_test.h"
 #endif

intern/cycles/util/util_math.h

@@ -788,7 +788,7 @@ ccl_device_inline float compare_floats(float a, float b, float abs_diff, int ulp
 }
 
 /* Calculate the angle between the two vectors a and b.
- * The usual approach acos(dot(a, b)) has severe precision issues for small angles,
+ * The usual approach `acos(dot(a, b))` has severe precision issues for small angles,
  * which are avoided by this method.
  * Based on "Mangled Angles" from https://people.eecs.berkeley.edu/~wkahan/Mindless.pdf
  */

intern/cycles/util/util_system.cpp

@@ -145,7 +145,8 @@ int system_cpu_num_active_group_processors()
   return numaAPI_GetNumCurrentNodesProcessors();
 }
 
-#if !defined(_WIN32) || defined(FREE_WINDOWS)
+/* Equivalent of Windows __cpuid for x86 processors on other platforms. */
+#if (!defined(_WIN32) || defined(FREE_WINDOWS)) && (defined(__x86_64__) || defined(__i386__))
 static void __cpuid(int data[4], int selector)
 {
 #  if defined(__x86_64__)
@@ -166,24 +167,54 @@ static void __cpuid(int data[4], int selector)
 
 string system_cpu_brand_string()
 {
+#if defined(__APPLE__)
+  /* Get from system on macOS. */
+  char modelname[512] = "";
+  size_t bufferlen = 512;
+  if (sysctlbyname("machdep.cpu.brand_string", &modelname, &bufferlen, NULL, 0) == 0) {
+    return modelname;
+  }
+#elif defined(WIN32) || defined(__x86_64__) || defined(__i386__)
+  /* Get from intrinsics on Windows and x86. */
   char buf[49] = {0};
   int result[4] = {0};
 
   __cpuid(result, 0x80000000);
 
-  if (result[0] >= (int)0x80000004) {
+  if (result[0] != 0 && result[0] >= (int)0x80000004) {
     __cpuid((int *)(buf + 0), 0x80000002);
     __cpuid((int *)(buf + 16), 0x80000003);
     __cpuid((int *)(buf + 32), 0x80000004);
 
     string brand = buf;
 
-    /* make it a bit more presentable */
+    /* Make it a bit more presentable. */
     brand = string_remove_trademark(brand);
 
     return brand;
   }
+#else
+  /* Get from /proc/cpuinfo on Unix systems. */
+  FILE *cpuinfo = fopen("/proc/cpuinfo", "r");
+  if (cpuinfo != nullptr) {
+    char cpuinfo_buf[513] = "";
+    fread(cpuinfo_buf, sizeof(cpuinfo_buf) - 1, 1, cpuinfo);
+    fclose(cpuinfo);
 
+    char *modelname = strstr(cpuinfo_buf, "model name");
+    if (modelname != nullptr) {
+      modelname = strchr(modelname, ':');
+      if (modelname != nullptr) {
+        modelname += 2;
+        char *modelname_end = strchr(modelname, '\n');
+        if (modelname_end != nullptr) {
+          *modelname_end = '\0';
+          return modelname;
+        }
+      }
+    }
+  }
+#endif
   return "Unknown CPU";
 }
 
@@ -192,7 +223,7 @@ int system_cpu_bits()
   return (sizeof(void *) * 8);
 }
 
-#if defined(__x86_64__) || defined(_M_X64) || defined(i386) || defined(_M_IX86)
+#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
 
 struct CPUCapabilities {
   bool x64;

intern/cycles/util/util_vector.h

@@ -43,8 +43,8 @@ class vector : public std::vector<value_type, allocator_type> {
   /* Try as hard as possible to use zero memory. */
   void free_memory()
   {
-    BaseClass::resize(0);
-    BaseClass::shrink_to_fit();
+    vector<value_type, allocator_type> empty;
+    BaseClass::swap(empty);
   }
 
   /* Some external API might demand working with std::vector. */

intern/ffmpeg/ffmpeg_compat.h

@@ -22,10 +22,17 @@
 
 #include <libavformat/avformat.h>
 
-/* check our ffmpeg is new enough, avoids user complaints */
-#if (LIBAVFORMAT_VERSION_MAJOR < 52) || \
-    ((LIBAVFORMAT_VERSION_MAJOR == 52) && (LIBAVFORMAT_VERSION_MINOR <= 64))
-#  error "FFmpeg 0.7 or newer is needed, Upgrade your FFmpeg or disable it"
+/* Check if our ffmpeg is new enough, avoids user complaints.
+ * Minimum supported version is currently 3.2.0 which mean the following library versions:
+ * libavutil   > 55.30
+ * libavcodec  > 57.60
+ * libavformat > 57.50
+ *
+ * We only check for one of these as they are usually updated in tandem.
+ */
+#if (LIBAVFORMAT_VERSION_MAJOR < 57) || \
+    ((LIBAVFORMAT_VERSION_MAJOR == 57) && (LIBAVFORMAT_VERSION_MINOR <= 50))
+#  error "FFmpeg 3.2.0 or newer is needed, Upgrade your FFmpeg or disable it"
 #endif
 /* end sanity check */
 
@@ -36,520 +43,83 @@
 #  define FFMPEG_INLINE static inline
 #endif
 
-#include <libavcodec/avcodec.h>
-#include <libavutil/mathematics.h>
-#include <libavutil/opt.h>
-#include <libavutil/rational.h>
+#if (LIBAVFORMAT_VERSION_MAJOR < 58) || \
+    ((LIBAVFORMAT_VERSION_MAJOR == 58) && (LIBAVFORMAT_VERSION_MINOR < 76))
+#  define FFMPEG_USE_DURATION_WORKAROUND 1
 
-#if (LIBAVFORMAT_VERSION_MAJOR > 52) || \
-    ((LIBAVFORMAT_VERSION_MAJOR >= 52) && (LIBAVFORMAT_VERSION_MINOR >= 101))
-#  define FFMPEG_HAVE_PARSE_UTILS 1
-#  include <libavutil/parseutils.h>
-#endif
-
-#include <libswscale/swscale.h>
-
-#if (LIBAVFORMAT_VERSION_MAJOR > 52) || \
-    ((LIBAVFORMAT_VERSION_MAJOR >= 52) && (LIBAVFORMAT_VERSION_MINOR >= 105))
-#  define FFMPEG_HAVE_AVIO 1
-#endif
-
-#if (LIBAVCODEC_VERSION_MAJOR > 53) || \
-    ((LIBAVCODEC_VERSION_MAJOR == 53) && (LIBAVCODEC_VERSION_MINOR > 1)) || \
-    ((LIBAVCODEC_VERSION_MAJOR == 53) && (LIBAVCODEC_VERSION_MINOR == 1) && \
-     (LIBAVCODEC_VERSION_MICRO >= 1)) || \
-    ((LIBAVCODEC_VERSION_MAJOR == 52) && (LIBAVCODEC_VERSION_MINOR >= 121))
-#  define FFMPEG_HAVE_DEFAULT_VAL_UNION 1
-#endif
-
-#if (LIBAVFORMAT_VERSION_MAJOR > 52) || \
-    ((LIBAVFORMAT_VERSION_MAJOR >= 52) && (LIBAVFORMAT_VERSION_MINOR >= 101))
-#  define FFMPEG_HAVE_AV_DUMP_FORMAT 1
-#endif
-
-#if (LIBAVFORMAT_VERSION_MAJOR > 52) || \
-    ((LIBAVFORMAT_VERSION_MAJOR >= 52) && (LIBAVFORMAT_VERSION_MINOR >= 45))
-#  define FFMPEG_HAVE_AV_GUESS_FORMAT 1
-#endif
-
-#if (LIBAVCODEC_VERSION_MAJOR > 52) || \
-    ((LIBAVCODEC_VERSION_MAJOR >= 52) && (LIBAVCODEC_VERSION_MINOR >= 23))
-#  define FFMPEG_HAVE_DECODE_AUDIO3 1
-#  define FFMPEG_HAVE_DECODE_VIDEO2 1
-#endif
-
-#if (LIBAVCODEC_VERSION_MAJOR > 52) || \
-    ((LIBAVCODEC_VERSION_MAJOR >= 52) && (LIBAVCODEC_VERSION_MINOR >= 64))
-#  define FFMPEG_HAVE_AVMEDIA_TYPES 1
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 52) || \
-     (LIBAVCODEC_VERSION_MAJOR >= 52) && (LIBAVCODEC_VERSION_MINOR >= 29)) && \
-    ((LIBSWSCALE_VERSION_MAJOR > 0) || \
-     (LIBSWSCALE_VERSION_MAJOR >= 0) && (LIBSWSCALE_VERSION_MINOR >= 10))
-#  define FFMPEG_SWSCALE_COLOR_SPACE_SUPPORT
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 54) || \
-     (LIBAVCODEC_VERSION_MAJOR >= 54) && (LIBAVCODEC_VERSION_MINOR > 14))
-#  define FFMPEG_HAVE_CANON_H264_RESOLUTION_FIX
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 53) || \
-     (LIBAVCODEC_VERSION_MAJOR >= 53) && (LIBAVCODEC_VERSION_MINOR >= 60))
-#  define FFMPEG_HAVE_ENCODE_AUDIO2
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 53) || \
-     (LIBAVCODEC_VERSION_MAJOR >= 53) && (LIBAVCODEC_VERSION_MINOR >= 42))
-#  define FFMPEG_HAVE_DECODE_AUDIO4
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 54) || \
-     (LIBAVCODEC_VERSION_MAJOR >= 54) && (LIBAVCODEC_VERSION_MINOR >= 13))
-#  define FFMPEG_HAVE_AVFRAME_SAMPLE_RATE
-#endif
-
-#if ((LIBAVUTIL_VERSION_MAJOR > 51) || \
-     (LIBAVUTIL_VERSION_MAJOR == 51) && (LIBAVUTIL_VERSION_MINOR >= 21))
-#  define FFMPEG_FFV1_ALPHA_SUPPORTED
-#  define FFMPEG_SAMPLE_FMT_S16P_SUPPORTED
-#else
+/* Before ffmpeg 4.4, package duration calculation used depricated variables to calculate the
+ * packet duration. Use the function from commit
+ * github.com/FFmpeg/FFmpeg/commit/1c0885334dda9ee8652e60c586fa2e3674056586
+ * to calculate the correct framerate for ffmpeg < 4.4.
+ */
 
 FFMPEG_INLINE
-int av_sample_fmt_is_planar(enum AVSampleFormat sample_fmt)
+void my_guess_pkt_duration(AVFormatContext *s, AVStream *st, AVPacket *pkt)
 {
-  /* no planar formats in FFmpeg < 0.9 */
-  (void)sample_fmt;
-  return 0;
-}
-
-#endif
-
-/* XXX TODO Probably fix to correct modern flags in code? Not sure how old FFMPEG we want to
- * support though, so for now this will do. */
-
-#ifndef FF_MIN_BUFFER_SIZE
-#  ifdef AV_INPUT_BUFFER_MIN_SIZE
-#    define FF_MIN_BUFFER_SIZE AV_INPUT_BUFFER_MIN_SIZE
-#  endif
-#endif
-
-#ifndef FF_INPUT_BUFFER_PADDING_SIZE
-#  ifdef AV_INPUT_BUFFER_PADDING_SIZE
-#    define FF_INPUT_BUFFER_PADDING_SIZE AV_INPUT_BUFFER_PADDING_SIZE
-#  endif
-#endif
-
-#ifndef CODEC_FLAG_GLOBAL_HEADER
-#  ifdef AV_CODEC_FLAG_GLOBAL_HEADER
-#    define CODEC_FLAG_GLOBAL_HEADER AV_CODEC_FLAG_GLOBAL_HEADER
-#  endif
-#endif
-
-#ifndef CODEC_FLAG_GLOBAL_HEADER
-#  ifdef AV_CODEC_FLAG_GLOBAL_HEADER
-#    define CODEC_FLAG_GLOBAL_HEADER AV_CODEC_FLAG_GLOBAL_HEADER
-#  endif
-#endif
-
-#ifndef CODEC_FLAG_INTERLACED_DCT
-#  ifdef AV_CODEC_FLAG_INTERLACED_DCT
-#    define CODEC_FLAG_INTERLACED_DCT AV_CODEC_FLAG_INTERLACED_DCT
-#  endif
-#endif
-
-#ifndef CODEC_FLAG_INTERLACED_ME
-#  ifdef AV_CODEC_FLAG_INTERLACED_ME
-#    define CODEC_FLAG_INTERLACED_ME AV_CODEC_FLAG_INTERLACED_ME
-#  endif
-#endif
-
-/* FFmpeg upstream 1.0 is the first who added AV_ prefix. */
-#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
-#  define AV_CODEC_ID_NONE CODEC_ID_NONE
-#  define AV_CODEC_ID_MPEG4 CODEC_ID_MPEG4
-#  define AV_CODEC_ID_MJPEG CODEC_ID_MJPEG
-#  define AV_CODEC_ID_DNXHD CODEC_ID_DNXHD
-#  define AV_CODEC_ID_MPEG2VIDEO CODEC_ID_MPEG2VIDEO
-#  define AV_CODEC_ID_MPEG1VIDEO CODEC_ID_MPEG1VIDEO
-#  define AV_CODEC_ID_DVVIDEO CODEC_ID_DVVIDEO
-#  define AV_CODEC_ID_THEORA CODEC_ID_THEORA
-#  define AV_CODEC_ID_PNG CODEC_ID_PNG
-#  define AV_CODEC_ID_QTRLE CODEC_ID_QTRLE
-#  define AV_CODEC_ID_FFV1 CODEC_ID_FFV1
-#  define AV_CODEC_ID_HUFFYUV CODEC_ID_HUFFYUV
-#  define AV_CODEC_ID_H264 CODEC_ID_H264
-#  define AV_CODEC_ID_FLV1 CODEC_ID_FLV1
-
-#  define AV_CODEC_ID_AAC CODEC_ID_AAC
-#  define AV_CODEC_ID_AC3 CODEC_ID_AC3
-#  define AV_CODEC_ID_MP3 CODEC_ID_MP3
-#  define AV_CODEC_ID_MP2 CODEC_ID_MP2
-#  define AV_CODEC_ID_FLAC CODEC_ID_FLAC
-#  define AV_CODEC_ID_PCM_U8 CODEC_ID_PCM_U8
-#  define AV_CODEC_ID_PCM_S16LE CODEC_ID_PCM_S16LE
-#  define AV_CODEC_ID_PCM_S24LE CODEC_ID_PCM_S24LE
-#  define AV_CODEC_ID_PCM_S32LE CODEC_ID_PCM_S32LE
-#  define AV_CODEC_ID_PCM_F32LE CODEC_ID_PCM_F32LE
-#  define AV_CODEC_ID_PCM_F64LE CODEC_ID_PCM_F64LE
-#  define AV_CODEC_ID_VORBIS CODEC_ID_VORBIS
-#endif
-
-FFMPEG_INLINE
-int av_get_cropped_height_from_codec(AVCodecContext *pCodecCtx)
-{
-  int y = pCodecCtx->height;
-
-#ifndef FFMPEG_HAVE_CANON_H264_RESOLUTION_FIX
-  /* really bad hack to remove this dreadfull black bar at the bottom
-   with Canon footage and old ffmpeg versions.
-   (to fix this properly in older ffmpeg versions one has to write a new
-   demuxer...)
-
-   see the actual fix here for reference:
-
-   http://git.libav.org/?p=libav.git;a=commit;h=30f515091c323da59c0f1b533703dedca2f4b95d
-
-   We do our best to apply this only to matching footage.
-*/
-  if (pCodecCtx->width == 1920 && pCodecCtx->height == 1088 &&
-      pCodecCtx->pix_fmt == PIX_FMT_YUVJ420P && pCodecCtx->codec_id == AV_CODEC_ID_H264) {
-    y = 1080;
+  if (pkt->duration < 0 && st->codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
+    av_log(s,
+           AV_LOG_WARNING,
+           "Packet with invalid duration %" PRId64 " in stream %d\n",
+           pkt->duration,
+           pkt->stream_index);
+    pkt->duration = 0;
   }
-#endif
 
-  return y;
-}
+  if (pkt->duration) {
+    return;
+  }
 
-#if ((LIBAVUTIL_VERSION_MAJOR < 51) || \
-     (LIBAVUTIL_VERSION_MAJOR == 51) && (LIBAVUTIL_VERSION_MINOR < 22))
-FFMPEG_INLINE
-int av_opt_set(void *obj, const char *name, const char *val, int search_flags)
-{
-  const AVOption *rv = NULL;
-  (void)search_flags;
-  av_set_string3(obj, name, val, 1, &rv);
-  return rv != NULL;
-}
-
-FFMPEG_INLINE
-int av_opt_set_int(void *obj, const char *name, int64_t val, int search_flags)
-{
-  const AVOption *rv = NULL;
-  (void)search_flags;
-  rv = av_set_int(obj, name, val);
-  return rv != NULL;
-}
-
-FFMPEG_INLINE
-int av_opt_set_double(void *obj, const char *name, double val, int search_flags)
-{
-  const AVOption *rv = NULL;
-  (void)search_flags;
-  rv = av_set_double(obj, name, val);
-  return rv != NULL;
-}
-
-#  define AV_OPT_TYPE_INT FF_OPT_TYPE_INT
-#  define AV_OPT_TYPE_INT64 FF_OPT_TYPE_INT64
-#  define AV_OPT_TYPE_STRING FF_OPT_TYPE_STRING
-#  define AV_OPT_TYPE_CONST FF_OPT_TYPE_CONST
-#  define AV_OPT_TYPE_DOUBLE FF_OPT_TYPE_DOUBLE
-#  define AV_OPT_TYPE_FLOAT FF_OPT_TYPE_FLOAT
-#endif
-
-#if ((LIBAVUTIL_VERSION_MAJOR < 51) || \
-     (LIBAVUTIL_VERSION_MAJOR == 51) && (LIBAVUTIL_VERSION_MINOR < 54))
-FFMPEG_INLINE
-enum AVSampleFormat av_get_packed_sample_fmt(enum AVSampleFormat sample_fmt)
-{
-  if (sample_fmt < 0 || sample_fmt >= AV_SAMPLE_FMT_NB)
-    return AV_SAMPLE_FMT_NONE;
-  return sample_fmt;
-}
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR < 53) || \
-     (LIBAVCODEC_VERSION_MAJOR == 53 && LIBAVCODEC_VERSION_MINOR < 35))
-FFMPEG_INLINE
-int avcodec_open2(AVCodecContext *avctx, AVCodec *codec, AVDictionary **options)
-{
-  /* TODO: no options are taking into account */
-  (void)options;
-  return avcodec_open(avctx, codec);
-}
-#endif
-
-#if ((LIBAVFORMAT_VERSION_MAJOR < 53) || \
-     (LIBAVFORMAT_VERSION_MAJOR == 53 && LIBAVFORMAT_VERSION_MINOR < 21))
-FFMPEG_INLINE
-AVStream *avformat_new_stream(AVFormatContext *s, AVCodec *c)
-{
-  /* TODO: no codec is taking into account */
-  (void)c;
-  return av_new_stream(s, 0);
-}
-
-FFMPEG_INLINE
-int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
-{
-  /* TODO: no options are taking into account */
-  (void)options;
-  return av_find_stream_info(ic);
-}
-#endif
-
-#if ((LIBAVFORMAT_VERSION_MAJOR > 53) || \
-     ((LIBAVFORMAT_VERSION_MAJOR == 53) && (LIBAVFORMAT_VERSION_MINOR > 32)) || \
-     ((LIBAVFORMAT_VERSION_MAJOR == 53) && (LIBAVFORMAT_VERSION_MINOR == 24) && \
-      (LIBAVFORMAT_VERSION_MICRO >= 100)))
-FFMPEG_INLINE
-void my_update_cur_dts(AVFormatContext *s, AVStream *ref_st, int64_t timestamp)
-{
-  int i;
-
-  for (i = 0; i < s->nb_streams; i++) {
-    AVStream *st = s->streams[i];
-
-    st->cur_dts = av_rescale(timestamp,
-                             st->time_base.den * (int64_t)ref_st->time_base.num,
-                             st->time_base.num * (int64_t)ref_st->time_base.den);
+  switch (st->codecpar->codec_type) {
+    case AVMEDIA_TYPE_VIDEO:
+      if (st->avg_frame_rate.num > 0 && st->avg_frame_rate.den > 0) {
+        pkt->duration = av_rescale_q(1, av_inv_q(st->avg_frame_rate), st->time_base);
+      }
+      else if (st->time_base.num * 1000LL > st->time_base.den) {
+        pkt->duration = 1;
+      }
+      break;
+    case AVMEDIA_TYPE_AUDIO: {
+      int frame_size = av_get_audio_frame_duration2(st->codecpar, pkt->size);
+      if (frame_size && st->codecpar->sample_rate) {
+        pkt->duration = av_rescale_q(
+            frame_size, (AVRational){1, st->codecpar->sample_rate}, st->time_base);
+      }
+      break;
+    }
+    default:
+      break;
   }
 }
-
-FFMPEG_INLINE
-void av_update_cur_dts(AVFormatContext *s, AVStream *ref_st, int64_t timestamp)
-{
-  my_update_cur_dts(s, ref_st, timestamp);
-}
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR < 54) || \
-     (LIBAVCODEC_VERSION_MAJOR == 54 && LIBAVCODEC_VERSION_MINOR < 28))
-FFMPEG_INLINE
-void avcodec_free_frame(AVFrame **frame)
-{
-  /* don't need to do anything with old AVFrame
-   * since it does not have malloced members */
-  (void)frame;
-}
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 54) || \
-     (LIBAVCODEC_VERSION_MAJOR >= 54) && (LIBAVCODEC_VERSION_MINOR >= 13))
-#  define FFMPEG_HAVE_AVFRAME_SAMPLE_RATE
-#endif
-
-#if ((LIBAVCODEC_VERSION_MAJOR > 54) || \
-     (LIBAVCODEC_VERSION_MAJOR == 54 && LIBAVCODEC_VERSION_MINOR >= 13))
-#  define FFMPEG_HAVE_FRAME_CHANNEL_LAYOUT
-#endif
-
-#ifndef FFMPEG_HAVE_AVIO
-#  define AVIO_FLAG_WRITE URL_WRONLY
-#  define avio_open url_fopen
-#  define avio_tell url_ftell
-#  define avio_close url_fclose
-#  define avio_size url_fsize
-#endif
-
-/* There are some version in between, which have avio_... functions but no
- * AVIO_FLAG_... */
-#ifndef AVIO_FLAG_WRITE
-#  define AVIO_FLAG_WRITE URL_WRONLY
-#endif
-
-#ifndef AV_PKT_FLAG_KEY
-#  define AV_PKT_FLAG_KEY PKT_FLAG_KEY
-#endif
-
-#ifndef FFMPEG_HAVE_AV_DUMP_FORMAT
-#  define av_dump_format dump_format
-#endif
-
-#ifndef FFMPEG_HAVE_AV_GUESS_FORMAT
-#  define av_guess_format guess_format
-#endif
-
-#ifndef FFMPEG_HAVE_PARSE_UTILS
-#  define av_parse_video_rate av_parse_video_frame_rate
-#endif
-
-#ifdef FFMPEG_HAVE_DEFAULT_VAL_UNION
-#  define FFMPEG_DEF_OPT_VAL_INT(OPT) OPT->default_val.i64
-#  define FFMPEG_DEF_OPT_VAL_DOUBLE(OPT) OPT->default_val.dbl
-#else
-#  define FFMPEG_DEF_OPT_VAL_INT(OPT) OPT->default_val
-#  define FFMPEG_DEF_OPT_VAL_DOUBLE(OPT) OPT->default_val
-#endif
-
-#ifndef FFMPEG_HAVE_AVMEDIA_TYPES
-#  define AVMEDIA_TYPE_VIDEO CODEC_TYPE_VIDEO
-#  define AVMEDIA_TYPE_AUDIO CODEC_TYPE_AUDIO
-#endif
-
-#ifndef FFMPEG_HAVE_DECODE_AUDIO3
-FFMPEG_INLINE
-int avcodec_decode_audio3(AVCodecContext *avctx,
-                          int16_t *samples,
-                          int *frame_size_ptr,
-                          AVPacket *avpkt)
-{
-  return avcodec_decode_audio2(avctx, samples, frame_size_ptr, avpkt->data, avpkt->size);
-}
-#endif
-
-#ifndef FFMPEG_HAVE_DECODE_VIDEO2
-FFMPEG_INLINE
-int avcodec_decode_video2(AVCodecContext *avctx,
-                          AVFrame *picture,
-                          int *got_picture_ptr,
-                          AVPacket *avpkt)
-{
-  return avcodec_decode_video(avctx, picture, got_picture_ptr, avpkt->data, avpkt->size);
-}
 #endif
 
 FFMPEG_INLINE
-int64_t av_get_pts_from_frame(AVFormatContext *avctx, AVFrame *picture)
+int64_t timestamp_from_pts_or_dts(int64_t pts, int64_t dts)
 {
-  int64_t pts;
-#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 34, 100)
-  pts = picture->pts;
-#else
-  pts = picture->pkt_pts;
-#endif
-
+  /* Some videos do not have any pts values, use dts instead in those cases if
+   * possible. Usually when this happens dts can act as pts because as all frames
+   * should then be presented in their decoded in order. IE pts == dts. */
   if (pts == AV_NOPTS_VALUE) {
-    pts = picture->pkt_dts;
+    return dts;
   }
-  if (pts == AV_NOPTS_VALUE) {
-    pts = 0;
-  }
-
-  (void)avctx;
   return pts;
 }
 
-/* obsolete constant formerly defined in FFMpeg libavcodec/avcodec.h */
-#ifndef AVCODEC_MAX_AUDIO_FRAME_SIZE
-#  define AVCODEC_MAX_AUDIO_FRAME_SIZE 192000  // 1 second of 48khz 32bit audio
-#endif
-
-#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 1, 0)
 FFMPEG_INLINE
-int avcodec_encode_video2(AVCodecContext *avctx,
-                          AVPacket *pkt,
-                          const AVFrame *frame,
-                          int *got_output)
+int64_t av_get_pts_from_frame(AVFrame *picture)
 {
-  int outsize, ret;
-
-  ret = av_new_packet(pkt, avctx->width * avctx->height * 7 + 10000);
-  if (ret < 0)
-    return ret;
-
-  outsize = avcodec_encode_video(avctx, pkt->data, pkt->size, frame);
-  if (outsize <= 0) {
-    *got_output = 0;
-    av_free_packet(pkt);
-  }
-  else {
-    *got_output = 1;
-    av_shrink_packet(pkt, outsize);
-    if (avctx->coded_frame) {
-      pkt->pts = avctx->coded_frame->pts;
-      if (avctx->coded_frame->key_frame)
-        pkt->flags |= AV_PKT_FLAG_KEY;
-    }
-  }
-
-  return outsize >= 0 ? 0 : outsize;
+  return timestamp_from_pts_or_dts(picture->pts, picture->pkt_dts);
 }
 
-#endif
+/* -------------------------------------------------------------------- */
+/** \name Deinterlace code block
+ *
+ * NOTE: The code in this block are from FFmpeg 2.6.4, which is licensed by LGPL.
+ * \{ */
 
-#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(53, 17, 0)
-FFMPEG_INLINE
-void avformat_close_input(AVFormatContext **ctx)
-{
-  av_close_input_file(*ctx);
-  *ctx = NULL;
-}
-#endif
+#define MAX_NEG_CROP 1024
 
-#if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 8, 0)
-FFMPEG_INLINE
-AVFrame *av_frame_alloc(void)
-{
-  return avcodec_alloc_frame();
-}
-
-FFMPEG_INLINE
-void av_frame_free(AVFrame **frame)
-{
-  av_freep(frame);
-}
-#endif
-
-FFMPEG_INLINE
-const char *av_get_metadata_key_value(AVDictionary *metadata, const char *key)
-{
-  if (metadata == NULL) {
-    return NULL;
-  }
-  AVDictionaryEntry *tag = NULL;
-  while ((tag = av_dict_get(metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
-    if (!strcmp(tag->key, key)) {
-      return tag->value;
-    }
-  }
-  return NULL;
-}
-
-FFMPEG_INLINE
-bool av_check_encoded_with_ffmpeg(AVFormatContext *ctx)
-{
-  const char *encoder = av_get_metadata_key_value(ctx->metadata, "ENCODER");
-  if (encoder != NULL && !strncmp(encoder, "Lavf", 4)) {
-    return true;
-  }
-  return false;
-}
-
-#if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(51, 32, 0)
-#  define AV_OPT_SEARCH_FAKE_OBJ 0
-#endif
-
-#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
-#  define FFMPEG_HAVE_DEPRECATED_FLAGS2
-#endif
-
-/* Since FFmpeg-1.1 this constant have AV_ prefix. */
-#if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 3, 100)
-#  define AV_PIX_FMT_BGR32 PIX_FMT_BGR32
-#  define AV_PIX_FMT_YUV422P PIX_FMT_YUV422P
-#  define AV_PIX_FMT_BGRA PIX_FMT_BGRA
-#  define AV_PIX_FMT_ARGB PIX_FMT_ARGB
-#  define AV_PIX_FMT_RGBA PIX_FMT_RGBA
-#endif
-
-/* New API from FFmpeg-2.0 which soon became recommended one. */
-#if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 38, 100)
-#  define av_frame_alloc avcodec_alloc_frame
-#  define av_frame_free avcodec_free_frame
-#  define av_frame_unref avcodec_get_frame_defaults
-#endif
-
-#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 24, 102)
-
-/* NOTE: The code in this block are from FFmpeg 2.6.4, which is licensed by LGPL. */
-
-#  define MAX_NEG_CROP 1024
-
-#  define times4(x) x, x, x, x
-#  define times256(x) times4(times4(times4(times4(times4(x)))))
+#define times4(x) x, x, x, x
+#define times256(x) times4(times4(times4(times4(times4(x)))))
 
 static const uint8_t ff_compat_crop_tab[256 + 2 * MAX_NEG_CROP] = {
     times256(0x00), 0x00, 0x01, 0x02, 0x03, 0x04,          0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
@@ -575,8 +145,8 @@ static const uint8_t ff_compat_crop_tab[256 + 2 * MAX_NEG_CROP] = {
     0xEF,           0xF0, 0xF1, 0xF2, 0xF3, 0xF4,          0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA,
     0xFB,           0xFC, 0xFD, 0xFE, 0xFF, times256(0xFF)};
 
-#  undef times4
-#  undef times256
+#undef times4
+#undef times256
 
 /* filter parameters: [-1 4 2 4 -1] // 8 */
 FFMPEG_INLINE
@@ -668,8 +238,9 @@ int deinterlace_bottom_field_inplace(uint8_t *src1, int src_wrap, int width, int
   uint8_t *src_m1, *src_0, *src_p1, *src_p2;
   int y;
   uint8_t *buf = (uint8_t *)av_malloc(width);
-  if (!buf)
+  if (!buf) {
     return AVERROR(ENOMEM);
+  }
 
   src_m1 = src1;
   memcpy(buf, src_m1, width);
@@ -689,24 +260,21 @@ int deinterlace_bottom_field_inplace(uint8_t *src1, int src_wrap, int width, int
   return 0;
 }
 
-#  ifdef __GNUC__
-#    pragma GCC diagnostic push
-#    pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#  endif
-
 FFMPEG_INLINE
-int avpicture_deinterlace(
-    AVPicture *dst, const AVPicture *src, enum AVPixelFormat pix_fmt, int width, int height)
+int av_image_deinterlace(
+    AVFrame *dst, const AVFrame *src, enum AVPixelFormat pix_fmt, int width, int height)
 {
   int i, ret;
 
   if (pix_fmt != AV_PIX_FMT_YUV420P && pix_fmt != AV_PIX_FMT_YUVJ420P &&
       pix_fmt != AV_PIX_FMT_YUV422P && pix_fmt != AV_PIX_FMT_YUVJ422P &&
       pix_fmt != AV_PIX_FMT_YUV444P && pix_fmt != AV_PIX_FMT_YUV411P &&
-      pix_fmt != AV_PIX_FMT_GRAY8)
+      pix_fmt != AV_PIX_FMT_GRAY8) {
     return -1;
-  if ((width & 3) != 0 || (height & 3) != 0)
+  }
+  if ((width & 3) != 0 || (height & 3) != 0) {
     return -1;
+  }
 
   for (i = 0; i < 3; i++) {
     if (i == 1) {
@@ -732,8 +300,9 @@ int avpicture_deinterlace(
     }
     if (src == dst) {
       ret = deinterlace_bottom_field_inplace(dst->data[i], dst->linesize[i], width, height);
-      if (ret < 0)
+      if (ret < 0) {
         return ret;
+      }
     }
     else {
       deinterlace_bottom_field(
@@ -743,10 +312,6 @@ int avpicture_deinterlace(
   return 0;
 }
 
-#  ifdef __GNUC__
-#    pragma GCC diagnostic pop
-#  endif
-
-#endif
+/* --- Deinterlace code block end --- */
 
 #endif