support anonymous attributes in foreach_attribute

The cache is used to fill the buffer list.

CMakeLists.txt

@@ -1598,6 +1598,9 @@ elseif(CMAKE_C_COMPILER_ID MATCHES "Clang")
   ADD_CHECK_C_COMPILER_FLAG(C_WARNINGS C_WARN_UNUSED_PARAMETER -Wunused-parameter)
 
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_ALL -Wall)
+  # Using C++20 features while having C++17 as the project language isn't allowed by MSVC.
+  ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_CXX20_DESIGNATOR -Wc++20-designator)
+
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_AUTOLOGICAL_COMPARE -Wno-tautological-compare)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_UNKNOWN_PRAGMAS     -Wno-unknown-pragmas)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_CHAR_SUBSCRIPTS     -Wno-char-subscripts)

build_files/cmake/Modules/FindZstd.cmake

@@ -0,0 +1,66 @@
+# - Find Zstd library
+# Find the native Zstd includes and library
+# This module defines
+#  ZSTD_INCLUDE_DIRS, where to find zstd.h, Set when
+#                     ZSTD_INCLUDE_DIR is found.
+#  ZSTD_LIBRARIES, libraries to link against to use Zstd.
+#  ZSTD_ROOT_DIR, The base directory to search for Zstd.
+#                 This can also be an environment variable.
+#  ZSTD_FOUND, If false, do not try to use Zstd.
+#
+# also defined, but not for general use are
+#  ZSTD_LIBRARY, where to find the Zstd library.
+
+#=============================================================================
+# Copyright 2019 Blender Foundation.
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+
+# If ZSTD_ROOT_DIR was defined in the environment, use it.
+IF(NOT ZSTD_ROOT_DIR AND NOT $ENV{ZSTD_ROOT_DIR} STREQUAL "")
+  SET(ZSTD_ROOT_DIR $ENV{ZSTD_ROOT_DIR})
+ENDIF()
+
+SET(_zstd_SEARCH_DIRS
+  ${ZSTD_ROOT_DIR}
+)
+
+FIND_PATH(ZSTD_INCLUDE_DIR
+  NAMES
+    zstd.h
+  HINTS
+    ${_zstd_SEARCH_DIRS}
+  PATH_SUFFIXES
+    include
+)
+
+FIND_LIBRARY(ZSTD_LIBRARY
+  NAMES
+    zstd
+  HINTS
+    ${_zstd_SEARCH_DIRS}
+  PATH_SUFFIXES
+    lib64 lib
+  )
+
+# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(Zstd DEFAULT_MSG
+    ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+IF(ZSTD_FOUND)
+  SET(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+  SET(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+ENDIF()
+
+MARK_AS_ADVANCED(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY
+)

build_files/cmake/platform/platform_apple.cmake

@@ -441,6 +441,9 @@ if(WITH_HARU)
   endif()
 endif()
 
+set(ZSTD_ROOT_DIR ${LIBDIR}/zstd)
+find_package(Zstd REQUIRED)
+
 if(EXISTS ${LIBDIR})
   without_system_libs_end()
 endif()

build_files/cmake/platform/platform_unix.cmake

@@ -99,6 +99,7 @@ endif()
 find_package_wrapper(JPEG REQUIRED)
 find_package_wrapper(PNG REQUIRED)
 find_package_wrapper(ZLIB REQUIRED)
+find_package_wrapper(Zstd REQUIRED)
 find_package_wrapper(Freetype REQUIRED)
 
 if(WITH_PYTHON)

build_files/cmake/platform/platform_win32.cmake

@@ -873,3 +873,6 @@ if(WITH_HARU)
     set(WITH_HARU OFF)
   endif()
 endif()
+
+set(ZSTD_INCLUDE_DIRS ${LIBDIR}/zstd/include)
+set(ZSTD_LIBRARIES ${LIBDIR}/zstd/lib/zstd_static.lib)

build_files/config/pipeline_config.yaml

@@ -51,9 +51,9 @@ buildbot:
     gcc:
         version: '9.0.0'
     cuda10:
-        version: '10.1.0'
+        version: '10.1.243'
     cuda11:
-        version: '11.4.0'
+        version: '11.4.1'
     optix:
         version: '7.1.0'
     cmake:

doc/python_api/requirements.txt

@@ -1,13 +1,13 @@
-sphinx==3.5.4
+sphinx==4.1.1
 
 # Sphinx dependencies that are important
-Jinja2==2.11.3
-Pygments==2.9.0
-docutils==0.16
+Jinja2==3.0.1
+Pygments==2.10.0
+docutils==0.17.1
 snowballstemmer==2.1.0
 babel==2.9.1
-requests==2.25.1
+requests==2.26.0
 
 # Only needed to match the theme used for the official documentation.
 # Without this theme, the default theme will be used.
-sphinx_rtd_theme==0.5.2
+sphinx_rtd_theme==1.0.0rc1

intern/cycles/blender/addon/properties.py

@@ -408,7 +408,7 @@ class CyclesRenderSettings(bpy.types.PropertyGroup):
 
     adaptive_threshold: FloatProperty(
         name="Adaptive Sampling Threshold",
-        description="Noise level step to stop sampling at, lower values reduce noise the cost of render time. Zero for automatic setting based on number of AA samples",
+        description="Noise level step to stop sampling at, lower values reduce noise at the cost of render time. Zero for automatic setting based on number of AA samples",
         min=0.0, max=1.0,
         default=0.0,
         precision=4,

intern/cycles/blender/blender_object.cpp

@@ -523,6 +523,9 @@ void BlenderSync::sync_procedural(BL::Object &b_ob,
 
   procedural->set_scale(cache_file.scale());
 
+  procedural->set_use_prefetch(cache_file.use_prefetch());
+  procedural->set_prefetch_cache_size(cache_file.prefetch_cache_size());
+
   /* create or update existing AlembicObjects */
   ustring object_path = ustring(b_mesh_cache.object_path());
 

intern/cycles/blender/blender_util.h

@@ -424,7 +424,7 @@ static inline void set_enum(PointerRNA &ptr, const char *name, const string &ide
 static inline string get_string(PointerRNA &ptr, const char *name)
 {
   char cstrbuf[1024];
-  char *cstr = RNA_string_get_alloc(&ptr, name, cstrbuf, sizeof(cstrbuf));
+  char *cstr = RNA_string_get_alloc(&ptr, name, cstrbuf, sizeof(cstrbuf), NULL);
   string str(cstr);
   if (cstr != cstrbuf)
     MEM_freeN(cstr);

intern/cycles/kernel/closure/bsdf_principled_diffuse.h

@@ -36,10 +36,10 @@ static_assert(sizeof(ShaderClosure) >= sizeof(PrincipledDiffuseBsdf),
 ccl_device float3 calculate_principled_diffuse_brdf(
     const PrincipledDiffuseBsdf *bsdf, float3 N, float3 V, float3 L, float3 H, float *pdf)
 {
-  float NdotL = max(dot(N, L), 0.0f);
-  float NdotV = max(dot(N, V), 0.0f);
+  float NdotL = dot(N, L);
+  float NdotV = dot(N, V);
 
-  if (NdotL < 0 || NdotV < 0) {
+  if (NdotL <= 0 || NdotV <= 0) {
     *pdf = 0.0f;
     return make_float3(0.0f, 0.0f, 0.0f);
   }

intern/cycles/render/alembic.cpp

@@ -25,6 +25,7 @@
 #include "render/shader.h"
 
 #include "util/util_foreach.h"
+#include "util/util_logging.h"
 #include "util/util_progress.h"
 #include "util/util_transform.h"
 #include "util/util_vector.h"
@@ -211,6 +212,35 @@ void CachedData::set_time_sampling(TimeSampling time_sampling)
   }
 }
 
+size_t CachedData::memory_used() const
+{
+  size_t mem_used = 0;
+
+  mem_used += curve_first_key.memory_used();
+  mem_used += curve_keys.memory_used();
+  mem_used += curve_radius.memory_used();
+  mem_used += curve_shader.memory_used();
+  mem_used += num_ngons.memory_used();
+  mem_used += shader.memory_used();
+  mem_used += subd_creases_edge.memory_used();
+  mem_used += subd_creases_weight.memory_used();
+  mem_used += subd_face_corners.memory_used();
+  mem_used += subd_num_corners.memory_used();
+  mem_used += subd_ptex_offset.memory_used();
+  mem_used += subd_smooth.memory_used();
+  mem_used += subd_start_corner.memory_used();
+  mem_used += transforms.memory_used();
+  mem_used += triangles.memory_used();
+  mem_used += uv_loops.memory_used();
+  mem_used += vertices.memory_used();
+
+  for (const CachedAttribute &attr : attributes) {
+    mem_used += attr.data.memory_used();
+  }
+
+  return mem_used;
+}
+
 static M44d convert_yup_zup(const M44d &mtx, float scale_mult)
 {
   V3d scale, shear, rotation, translation;
@@ -706,6 +736,9 @@ NODE_DEFINE(AlembicProcedural)
 
   SOCKET_NODE_ARRAY(objects, "Objects", AlembicObject::get_node_type());
 
+  SOCKET_BOOLEAN(use_prefetch, "Use Prefetch", true);
+  SOCKET_INT(prefetch_cache_size, "Prefetch Cache Size", 4096);
+
   return type;
 }
 
@@ -823,6 +856,30 @@ void AlembicProcedural::generate(Scene *scene, Progress &progress)
     }
   }
 
+  if (use_prefetch_is_modified()) {
+    if (!use_prefetch) {
+      for (Node *node : objects) {
+        AlembicObject *object = static_cast<AlembicObject *>(node);
+        object->clear_cache();
+      }
+    }
+  }
+
+  if (prefetch_cache_size_is_modified()) {
+    /* Check whether the current memory usage fits in the new requested size,
+     * abort the render if it is any higher. */
+    size_t memory_used = 0ul;
+    for (Node *node : objects) {
+      AlembicObject *object = static_cast<AlembicObject *>(node);
+      memory_used += object->get_cached_data().memory_used();
+    }
+
+    if (memory_used > get_prefetch_cache_size_in_bytes()) {
+      progress.set_error("Error: Alembic Procedural memory limit reached");
+      return;
+    }
+  }
+
   build_caches(progress);
 
   foreach (Node *node, objects) {
@@ -1300,6 +1357,8 @@ void AlembicProcedural::walk_hierarchy(
 
 void AlembicProcedural::build_caches(Progress &progress)
 {
+  size_t memory_used = 0;
+
   for (Node *node : objects) {
     AlembicObject *object = static_cast<AlembicObject *>(node);
 
@@ -1353,7 +1412,18 @@ void AlembicProcedural::build_caches(Progress &progress)
     if (scale_is_modified() || object->get_cached_data().transforms.size() == 0) {
       object->setup_transform_cache(object->get_cached_data(), scale);
     }
+
+    memory_used += object->get_cached_data().memory_used();
+
+    if (use_prefetch) {
+      if (memory_used > get_prefetch_cache_size_in_bytes()) {
+        progress.set_error("Error: Alembic Procedural memory limit reached");
+        return;
+      }
+    }
   }
+
+  VLOG(1) << "AlembicProcedural memory usage : " << string_human_readable_size(memory_used);
 }
 
 CCL_NAMESPACE_END

intern/cycles/render/alembic.h

@@ -272,6 +272,21 @@ template<typename T> class DataStore {
     node->set(*socket, value);
   }
 
+  size_t memory_used() const
+  {
+    if constexpr (is_array<T>::value) {
+      size_t mem_used = 0;
+
+      for (const T &array : data) {
+        mem_used += array.size() * sizeof(array[0]);
+      }
+
+      return mem_used;
+    }
+
+    return data.size() * sizeof(T);
+  }
+
  private:
   const TimeIndexPair &get_index_for_time(double time) const
   {
@@ -332,6 +347,8 @@ struct CachedData {
   void invalidate_last_loaded_time(bool attributes_only = false);
 
   void set_time_sampling(Alembic::AbcCoreAbstract::TimeSampling time_sampling);
+
+  size_t memory_used() const;
 };
 
 /* Representation of an Alembic object for the AlembicProcedural.
@@ -482,6 +499,13 @@ class AlembicProcedural : public Procedural {
    * software. */
   NODE_SOCKET_API(float, scale)
 
+  /* Cache controls */
+  NODE_SOCKET_API(bool, use_prefetch)
+
+  /* Memory limit for the cache, if the data does not fit within this limit, rendering is aborted.
+   */
+  NODE_SOCKET_API(int, prefetch_cache_size)
+
   AlembicProcedural();
   ~AlembicProcedural();
 
@@ -531,6 +555,12 @@ class AlembicProcedural : public Procedural {
   void read_subd(AlembicObject *abc_object, Alembic::AbcGeom::Abc::chrono_t frame_time);
 
   void build_caches(Progress &progress);
+
+  size_t get_prefetch_cache_size_in_bytes() const
+  {
+    /* prefetch_cache_size is in megabytes, so convert to bytes. */
+    return static_cast<size_t>(prefetch_cache_size) * 1024 * 1024;
+  }
 };
 
 CCL_NAMESPACE_END

intern/cycles/render/alembic_read.cpp

@@ -44,9 +44,19 @@ static set<chrono_t> get_relevant_sample_times(AlembicProcedural *proc,
     return result;
   }
 
-  // load the data for the entire animation
-  const double start_frame = static_cast<double>(proc->get_start_frame());
-  const double end_frame = static_cast<double>(proc->get_end_frame());
+  double start_frame;
+  double end_frame;
+
+  if (proc->get_use_prefetch()) {
+    // load the data for the entire animation
+    start_frame = static_cast<double>(proc->get_start_frame());
+    end_frame = static_cast<double>(proc->get_end_frame());
+  }
+  else {
+    // load the data for the current frame
+    start_frame = static_cast<double>(proc->get_frame());
+    end_frame = start_frame;
+  }
 
   const double frame_rate = static_cast<double>(proc->get_frame_rate());
   const double start_time = start_frame / frame_rate;

intern/ghost/GHOST_IEvent.h

@@ -32,10 +32,10 @@ class GHOST_IWindow;
 /**
  * Interface class for events received from GHOST.
  * You should not need to inherit this class. The system will pass these events
- * to the GHOST_IEventConsumer::processEvent() method of event consumers.<br>
- * Use the getType() method to retrieve the type of event and the getData()
+ * to the #GHOST_IEventConsumer::processEvent() method of event consumers.<br>
+ * Use the #getType() method to retrieve the type of event and the #getData()
  * method to get the event data out. Using the event type you can cast the
- * event data to the correct event dat structure.
+ * event data to the correct event data structure.
  * \see GHOST_IEventConsumer#processEvent
  * \see GHOST_TEventType
  */

intern/ghost/intern/GHOST_SystemWin32.cpp

@@ -1741,7 +1741,7 @@ LRESULT WINAPI GHOST_SystemWin32::s_wndProc(HWND hwnd, UINT msg, WPARAM wParam,
         case WM_MOUSELEAVE: {
           window->m_mousePresent = false;
           if (window->getTabletData().Active == GHOST_kTabletModeNone) {
-            processCursorEvent(window);
+            event = processCursorEvent(window);
           }
           GHOST_Wintab *wt = window->getWintab();
           if (wt) {

release/scripts/startup/bl_ui/space_userpref.py

@@ -997,7 +997,6 @@ class USERPREF_PT_theme_text_style(ThemePanel, CenterAlignMixIn, Panel):
         flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=True)
 
         col = flow.column()
-        col.row().prop(font_style, "font_kerning_style", expand=True)
         col.prop(font_style, "points")
 
         col = flow.column(align=True)

source/blender/blenfont/BLF_api.h

@@ -98,13 +98,13 @@ void BLF_batch_draw_flush(void);
 void BLF_batch_draw_end(void);
 
 /* Draw the string using the current font. */
-void BLF_draw_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+void BLF_draw_ex(int fontid, const char *str, size_t str_len, struct ResultBLF *r_info)
     ATTR_NONNULL(2);
-void BLF_draw(int fontid, const char *str, size_t len) ATTR_NONNULL(2);
-void BLF_draw_ascii_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+void BLF_draw(int fontid, const char *str, size_t str_len) ATTR_NONNULL(2);
+void BLF_draw_ascii_ex(int fontid, const char *str, size_t str_len, struct ResultBLF *r_info)
     ATTR_NONNULL(2);
-void BLF_draw_ascii(int fontid, const char *str, size_t len) ATTR_NONNULL(2);
-int BLF_draw_mono(int fontid, const char *str, size_t len, int cwidth) ATTR_NONNULL(2);
+void BLF_draw_ascii(int fontid, const char *str, size_t str_len) ATTR_NONNULL(2);
+int BLF_draw_mono(int fontid, const char *str, size_t str_len, int cwidth) ATTR_NONNULL(2);
 
 typedef bool (*BLF_GlyphBoundsFn)(const char *str,
                                   const size_t str_step_ofs,
@@ -116,43 +116,45 @@ typedef bool (*BLF_GlyphBoundsFn)(const char *str,
 
 void BLF_boundbox_foreach_glyph_ex(int fontid,
                                    const char *str,
-                                   size_t len,
+                                   size_t str_len,
                                    BLF_GlyphBoundsFn user_fn,
                                    void *user_data,
                                    struct ResultBLF *r_info) ATTR_NONNULL(2);
 void BLF_boundbox_foreach_glyph(int fontid,
                                 const char *str,
-                                size_t len,
+                                size_t str_len,
                                 BLF_GlyphBoundsFn user_fn,
                                 void *user_data) ATTR_NONNULL(2);
 
 /* Get the string byte offset that fits within a given width */
-size_t BLF_width_to_strlen(int fontid, const char *str, size_t len, float width, float *r_width)
-    ATTR_NONNULL(2);
+size_t BLF_width_to_strlen(
+    int fontid, const char *str, size_t str_len, float width, float *r_width) ATTR_NONNULL(2);
 /* Same as BLF_width_to_strlen but search from the string end */
-size_t BLF_width_to_rstrlen(int fontid, const char *str, size_t len, float width, float *r_width)
-    ATTR_NONNULL(2);
+size_t BLF_width_to_rstrlen(
+    int fontid, const char *str, size_t str_len, float width, float *r_width) ATTR_NONNULL(2);
 
 /* This function return the bounding box of the string
  * and are not multiplied by the aspect.
  */
 void BLF_boundbox_ex(int fontid,
                      const char *str,
-                     size_t len,
+                     size_t str_len,
                      struct rctf *box,
                      struct ResultBLF *r_info) ATTR_NONNULL(2);
-void BLF_boundbox(int fontid, const char *str, size_t len, struct rctf *box) ATTR_NONNULL();
+void BLF_boundbox(int fontid, const char *str, size_t str_len, struct rctf *box) ATTR_NONNULL();
 
 /* The next both function return the width and height
  * of the string, using the current font and both value
  * are multiplied by the aspect of the font.
  */
-float BLF_width_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+float BLF_width_ex(int fontid, const char *str, size_t str_len, struct ResultBLF *r_info)
     ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(2);
-float BLF_width(int fontid, const char *str, size_t len) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
-float BLF_height_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+float BLF_width(int fontid, const char *str, size_t str_len) ATTR_WARN_UNUSED_RESULT
+    ATTR_NONNULL();
+float BLF_height_ex(int fontid, const char *str, size_t str_len, struct ResultBLF *r_info)
     ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(2);
-float BLF_height(int fontid, const char *str, size_t len) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+float BLF_height(int fontid, const char *str, size_t str_len) ATTR_WARN_UNUSED_RESULT
+    ATTR_NONNULL();
 
 /* Return dimensions of the font without any sample text. */
 int BLF_height_max(int fontid) ATTR_WARN_UNUSED_RESULT;
@@ -163,8 +165,8 @@ float BLF_ascender(int fontid) ATTR_WARN_UNUSED_RESULT;
 /* The following function return the width and height of the string, but
  * just in one call, so avoid extra freetype2 stuff.
  */
-void BLF_width_and_height(int fontid, const char *str, size_t len, float *r_width, float *r_height)
-    ATTR_NONNULL();
+void BLF_width_and_height(
+    int fontid, const char *str, size_t str_len, float *r_width, float *r_height) ATTR_NONNULL();
 
 /* For fixed width fonts only, returns the width of a
  * character.
@@ -221,9 +223,9 @@ void BLF_buffer_col(int fontid, const float rgba[4]) ATTR_NONNULL(2);
 /* Draw the string into the buffer, this function draw in both buffer,
  * float and unsigned char _BUT_ it's not necessary set both buffer, NULL is valid here.
  */
-void BLF_draw_buffer_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+void BLF_draw_buffer_ex(int fontid, const char *str, size_t str_len, struct ResultBLF *r_info)
     ATTR_NONNULL(2);
-void BLF_draw_buffer(int fontid, const char *str, size_t len) ATTR_NONNULL(2);
+void BLF_draw_buffer(int fontid, const char *str, size_t str_len) ATTR_NONNULL(2);
 
 /* Add a path to the font dir paths. */
 void BLF_dir_add(const char *path) ATTR_NONNULL();
@@ -254,8 +256,9 @@ void BLF_default_dpi(int dpi);
 void BLF_default_set(int fontid);
 int BLF_default(void); /* get default font ID so we can pass it to other functions */
 /* Draw the string using the default font, size and dpi. */
-void BLF_draw_default(float x, float y, float z, const char *str, size_t len) ATTR_NONNULL();
-void BLF_draw_default_ascii(float x, float y, float z, const char *str, size_t len) ATTR_NONNULL();
+void BLF_draw_default(float x, float y, float z, const char *str, size_t str_len) ATTR_NONNULL();
+void BLF_draw_default_ascii(float x, float y, float z, const char *str, size_t str_len)
+    ATTR_NONNULL();
 /* Set size and DPI, and return default font ID. */
 int BLF_set_default(void);
 

source/blender/blenfont/intern/blf.c

@@ -108,7 +108,6 @@ void BLF_cache_clear(void)
     FontBLF *font = global_font[i];
     if (font) {
       blf_glyph_cache_clear(font);
-      blf_kerning_cache_clear(font);
     }
   }
 }
@@ -522,7 +521,7 @@ static void blf_draw_gl__end(FontBLF *font)
   }
 }
 
-void BLF_draw_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+void BLF_draw_ex(int fontid, const char *str, const size_t str_len, struct ResultBLF *r_info)
 {
   FontBLF *font = blf_get(fontid);
 
@@ -531,27 +530,27 @@ void BLF_draw_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_in
   if (font) {
     blf_draw_gl__start(font);
     if (font->flags & BLF_WORD_WRAP) {
-      blf_font_draw__wrap(font, str, len, r_info);
+      blf_font_draw__wrap(font, str, str_len, r_info);
     }
     else {
-      blf_font_draw(font, str, len, r_info);
+      blf_font_draw(font, str, str_len, r_info);
     }
     blf_draw_gl__end(font);
   }
 }
-void BLF_draw(int fontid, const char *str, size_t len)
+void BLF_draw(int fontid, const char *str, const size_t str_len)
 {
-  if (len == 0 || str[0] == '\0') {
+  if (str_len == 0 || str[0] == '\0') {
     return;
   }
 
   /* Avoid bgl usage to corrupt BLF drawing. */
   GPU_bgl_end();
 
-  BLF_draw_ex(fontid, str, len, NULL);
+  BLF_draw_ex(fontid, str, str_len, NULL);
 }
 
-void BLF_draw_ascii_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+void BLF_draw_ascii_ex(int fontid, const char *str, const size_t str_len, struct ResultBLF *r_info)
 {
   FontBLF *font = blf_get(fontid);
 
@@ -561,27 +560,27 @@ void BLF_draw_ascii_ex(int fontid, const char *str, size_t len, struct ResultBLF
     blf_draw_gl__start(font);
     if (font->flags & BLF_WORD_WRAP) {
       /* Use non-ASCII draw function for word-wrap. */
-      blf_font_draw__wrap(font, str, len, r_info);
+      blf_font_draw__wrap(font, str, str_len, r_info);
     }
     else {
-      blf_font_draw_ascii(font, str, len, r_info);
+      blf_font_draw_ascii(font, str, str_len, r_info);
     }
     blf_draw_gl__end(font);
   }
 }
 
-void BLF_draw_ascii(int fontid, const char *str, size_t len)
+void BLF_draw_ascii(int fontid, const char *str, const size_t str_len)
 {
-  if (len == 0 || str[0] == '\0') {
+  if (str_len == 0 || str[0] == '\0') {
     return;
   }
 
-  BLF_draw_ascii_ex(fontid, str, len, NULL);
+  BLF_draw_ascii_ex(fontid, str, str_len, NULL);
 }
 
-int BLF_draw_mono(int fontid, const char *str, size_t len, int cwidth)
+int BLF_draw_mono(int fontid, const char *str, const size_t str_len, int cwidth)
 {
-  if (len == 0 || str[0] == '\0') {
+  if (str_len == 0 || str[0] == '\0') {
     return 0;
   }
 
@@ -590,7 +589,7 @@ int BLF_draw_mono(int fontid, const char *str, size_t len, int cwidth)
 
   if (font) {
     blf_draw_gl__start(font);
-    columns = blf_font_draw_mono(font, str, len, cwidth);
+    columns = blf_font_draw_mono(font, str, str_len, cwidth);
     blf_draw_gl__end(font);
   }
 
@@ -607,7 +606,7 @@ int BLF_draw_mono(int fontid, const char *str, size_t len, int cwidth)
  */
 void BLF_boundbox_foreach_glyph_ex(int fontid,
                                    const char *str,
-                                   size_t len,
+                                   size_t str_len,
                                    BLF_GlyphBoundsFn user_fn,
                                    void *user_data,
                                    struct ResultBLF *r_info)
@@ -622,25 +621,26 @@ void BLF_boundbox_foreach_glyph_ex(int fontid,
       BLI_assert(0);
     }
     else {
-      blf_font_boundbox_foreach_glyph(font, str, len, user_fn, user_data, r_info);
+      blf_font_boundbox_foreach_glyph(font, str, str_len, user_fn, user_data, r_info);
     }
   }
 }
 
 void BLF_boundbox_foreach_glyph(
-    int fontid, const char *str, size_t len, BLF_GlyphBoundsFn user_fn, void *user_data)
+    int fontid, const char *str, const size_t str_len, BLF_GlyphBoundsFn user_fn, void *user_data)
 {
-  BLF_boundbox_foreach_glyph_ex(fontid, str, len, user_fn, user_data, NULL);
+  BLF_boundbox_foreach_glyph_ex(fontid, str, str_len, user_fn, user_data, NULL);
 }
 
-size_t BLF_width_to_strlen(int fontid, const char *str, size_t len, float width, float *r_width)
+size_t BLF_width_to_strlen(
+    int fontid, const char *str, const size_t str_len, float width, float *r_width)
 {
   FontBLF *font = blf_get(fontid);
 
   if (font) {
     const float xa = (font->flags & BLF_ASPECT) ? font->aspect[0] : 1.0f;
     size_t ret;
-    ret = blf_font_width_to_strlen(font, str, len, width / xa, r_width);
+    ret = blf_font_width_to_strlen(font, str, str_len, width / xa, r_width);
     if (r_width) {
       *r_width *= xa;
     }
@@ -653,14 +653,15 @@ size_t BLF_width_to_strlen(int fontid, const char *str, size_t len, float width,
   return 0;
 }
 
-size_t BLF_width_to_rstrlen(int fontid, const char *str, size_t len, float width, float *r_width)
+size_t BLF_width_to_rstrlen(
+    int fontid, const char *str, const size_t str_len, float width, float *r_width)
 {
   FontBLF *font = blf_get(fontid);
 
   if (font) {
     const float xa = (font->flags & BLF_ASPECT) ? font->aspect[0] : 1.0f;
     size_t ret;
-    ret = blf_font_width_to_rstrlen(font, str, len, width / xa, r_width);
+    ret = blf_font_width_to_rstrlen(font, str, str_len, width / xa, r_width);
     if (r_width) {
       *r_width *= xa;
     }
@@ -674,7 +675,7 @@ size_t BLF_width_to_rstrlen(int fontid, const char *str, size_t len, float width
 }
 
 void BLF_boundbox_ex(
-    int fontid, const char *str, size_t len, rctf *r_box, struct ResultBLF *r_info)
+    int fontid, const char *str, const size_t str_len, rctf *r_box, struct ResultBLF *r_info)
 {
   FontBLF *font = blf_get(fontid);
 
@@ -682,47 +683,48 @@ void BLF_boundbox_ex(
 
   if (font) {
     if (font->flags & BLF_WORD_WRAP) {
-      blf_font_boundbox__wrap(font, str, len, r_box, r_info);
+      blf_font_boundbox__wrap(font, str, str_len, r_box, r_info);
     }
     else {
-      blf_font_boundbox(font, str, len, r_box, r_info);
+      blf_font_boundbox(font, str, str_len, r_box, r_info);
     }
   }
 }
 
-void BLF_boundbox(int fontid, const char *str, size_t len, rctf *r_box)
+void BLF_boundbox(int fontid, const char *str, const size_t str_len, rctf *r_box)
 {
-  BLF_boundbox_ex(fontid, str, len, r_box, NULL);
+  BLF_boundbox_ex(fontid, str, str_len, r_box, NULL);
 }
 
-void BLF_width_and_height(int fontid, const char *str, size_t len, float *r_width, float *r_height)
+void BLF_width_and_height(
+    int fontid, const char *str, const size_t str_len, float *r_width, float *r_height)
 {
   FontBLF *font = blf_get(fontid);
 
   if (font) {
-    blf_font_width_and_height(font, str, len, r_width, r_height, NULL);
+    blf_font_width_and_height(font, str, str_len, r_width, r_height, NULL);
   }
   else {
     *r_width = *r_height = 0.0f;
   }
 }
 
-float BLF_width_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+float BLF_width_ex(int fontid, const char *str, const size_t str_len, struct ResultBLF *r_info)
 {
   FontBLF *font = blf_get(fontid);
 
   BLF_RESULT_CHECK_INIT(r_info);
 
   if (font) {
-    return blf_font_width(font, str, len, r_info);
+    return blf_font_width(font, str, str_len, r_info);
   }
 
   return 0.0f;
 }
 
-float BLF_width(int fontid, const char *str, size_t len)
+float BLF_width(int fontid, const char *str, const size_t str_len)
 {
-  return BLF_width_ex(fontid, str, len, NULL);
+  return BLF_width_ex(fontid, str, str_len, NULL);
 }
 
 float BLF_fixed_width(int fontid)
@@ -736,22 +738,22 @@ float BLF_fixed_width(int fontid)
   return 0.0f;
 }
 
-float BLF_height_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+float BLF_height_ex(int fontid, const char *str, const size_t str_len, struct ResultBLF *r_info)
 {
   FontBLF *font = blf_get(fontid);
 
   BLF_RESULT_CHECK_INIT(r_info);
 
   if (font) {
-    return blf_font_height(font, str, len, r_info);
+    return blf_font_height(font, str, str_len, r_info);
   }
 
   return 0.0f;
 }
 
-float BLF_height(int fontid, const char *str, size_t len)
+float BLF_height(int fontid, const char *str, const size_t str_len)
 {
-  return BLF_height_ex(fontid, str, len, NULL);
+  return BLF_height_ex(fontid, str, str_len, NULL);
 }
 
 int BLF_height_max(int fontid)
@@ -895,24 +897,27 @@ void blf_draw_buffer__end(void)
 {
 }
 
-void BLF_draw_buffer_ex(int fontid, const char *str, size_t len, struct ResultBLF *r_info)
+void BLF_draw_buffer_ex(int fontid,
+                        const char *str,
+                        const size_t str_len,
+                        struct ResultBLF *r_info)
 {
   FontBLF *font = blf_get(fontid);
 
   if (font && (font->buf_info.fbuf || font->buf_info.cbuf)) {
     blf_draw_buffer__start(font);
     if (font->flags & BLF_WORD_WRAP) {
-      blf_font_draw_buffer__wrap(font, str, len, r_info);
+      blf_font_draw_buffer__wrap(font, str, str_len, r_info);
     }
     else {
-      blf_font_draw_buffer(font, str, len, r_info);
+      blf_font_draw_buffer(font, str, str_len, r_info);
     }
     blf_draw_buffer__end();
   }
 }
-void BLF_draw_buffer(int fontid, const char *str, size_t len)
+void BLF_draw_buffer(int fontid, const char *str, const size_t str_len)
 {
-  BLF_draw_buffer_ex(fontid, str, len, NULL);
+  BLF_draw_buffer_ex(fontid, str, str_len, NULL);
 }
 
 char *BLF_display_name_from_file(const char *filename)

source/blender/blenfont/intern/blf_default.c

@@ -68,23 +68,23 @@ int BLF_set_default(void)
   return global_font_default;
 }
 
-void BLF_draw_default(float x, float y, float z, const char *str, size_t len)
+void BLF_draw_default(float x, float y, float z, const char *str, const size_t str_len)
 {
   ASSERT_DEFAULT_SET;
 
   const uiStyle *style = UI_style_get();
   BLF_size(global_font_default, style->widgetlabel.points, global_font_dpi);
   BLF_position(global_font_default, x, y, z);
-  BLF_draw(global_font_default, str, len);
+  BLF_draw(global_font_default, str, str_len);
 }
 
 /* same as above but call 'BLF_draw_ascii' */
-void BLF_draw_default_ascii(float x, float y, float z, const char *str, size_t len)
+void BLF_draw_default_ascii(float x, float y, float z, const char *str, const size_t str_len)
 {
   ASSERT_DEFAULT_SET;
 
   const uiStyle *style = UI_style_get();
   BLF_size(global_font_default, style->widgetlabel.points, global_font_dpi);
   BLF_position(global_font_default, x, y, z);
-  BLF_draw_ascii(global_font_default, str, len); /* XXX, use real length */
+  BLF_draw_ascii(global_font_default, str, str_len); /* XXX, use real length */
 }

source/blender/blenfont/intern/blf_glyph.c

@@ -55,56 +55,6 @@
 #include "BLI_math_vector.h"
 #include "BLI_strict_flags.h"
 
-KerningCacheBLF *blf_kerning_cache_find(FontBLF *font)
-{
-  return (KerningCacheBLF *)font->kerning_caches.first;
-}
-
-/* Create a new glyph cache for the current kerning mode. */
-KerningCacheBLF *blf_kerning_cache_new(FontBLF *font, GlyphCacheBLF *gc)
-{
-  KerningCacheBLF *kc = MEM_mallocN(sizeof(KerningCacheBLF), __func__);
-  kc->next = NULL;
-  kc->prev = NULL;
-
-  GlyphBLF *g_table[KERNING_CACHE_TABLE_SIZE];
-  for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) {
-    GlyphBLF *g = blf_glyph_search(gc, i);
-    if (UNLIKELY(g == NULL)) {
-      FT_UInt glyph_index = FT_Get_Char_Index(font->face, i);
-      g = blf_glyph_add(font, gc, glyph_index, i);
-    }
-    g_table[i] = g;
-  }
-
-  memset(kc->ascii_table, 0, sizeof(kc->ascii_table));
-  for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) {
-    GlyphBLF *g = g_table[i];
-    if (g == NULL) {
-      continue;
-    }
-    for (uint j = 0; j < KERNING_CACHE_TABLE_SIZE; j++) {
-      GlyphBLF *g_prev = g_table[j];
-      if (g_prev == NULL) {
-        continue;
-      }
-      FT_Vector delta;
-      if (FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_DEFAULT, &delta) == 0) {
-        kc->ascii_table[i][j] = (int)delta.x >> 6;
-      }
-    }
-  }
-
-  BLI_addhead(&font->kerning_caches, kc);
-  return kc;
-}
-
-void blf_kerning_cache_clear(FontBLF *font)
-{
-  font->kerning_cache = NULL;
-  BLI_freelistN(&font->kerning_caches);
-}
-
 GlyphCacheBLF *blf_glyph_cache_find(FontBLF *font, unsigned int size, unsigned int dpi)
 {
   GlyphCacheBLF *p;

source/blender/blenfont/intern/blf_internal.h

@@ -53,46 +53,55 @@ struct FontBLF *blf_font_new_from_mem(const char *name, const unsigned char *mem
 void blf_font_attach_from_mem(struct FontBLF *font, const unsigned char *mem, int mem_size);
 
 void blf_font_size(struct FontBLF *font, unsigned int size, unsigned int dpi);
-void blf_font_draw(struct FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info);
+void blf_font_draw(struct FontBLF *font,
+                   const char *str,
+                   size_t str_len,
+                   struct ResultBLF *r_info);
 void blf_font_draw__wrap(struct FontBLF *font,
                          const char *str,
-                         size_t len,
+                         size_t str_len,
                          struct ResultBLF *r_info);
 void blf_font_draw_ascii(struct FontBLF *font,
                          const char *str,
-                         size_t len,
+                         size_t str_len,
                          struct ResultBLF *r_info);
-int blf_font_draw_mono(struct FontBLF *font, const char *str, size_t len, int cwidth);
+int blf_font_draw_mono(struct FontBLF *font, const char *str, size_t str_len, int cwidth);
 void blf_font_draw_buffer(struct FontBLF *font,
                           const char *str,
-                          size_t len,
+                          size_t str_len,
                           struct ResultBLF *r_info);
 void blf_font_draw_buffer__wrap(struct FontBLF *font,
                                 const char *str,
-                                size_t len,
+                                size_t str_len,
                                 struct ResultBLF *r_info);
 size_t blf_font_width_to_strlen(
-    struct FontBLF *font, const char *str, size_t len, float width, float *r_width);
+    struct FontBLF *font, const char *str, size_t str_len, float width, float *r_width);
 size_t blf_font_width_to_rstrlen(
-    struct FontBLF *font, const char *str, size_t len, float width, float *r_width);
+    struct FontBLF *font, const char *str, size_t str_len, float width, float *r_width);
 void blf_font_boundbox(struct FontBLF *font,
                        const char *str,
-                       size_t len,
+                       size_t str_len,
                        struct rctf *r_box,
                        struct ResultBLF *r_info);
 void blf_font_boundbox__wrap(struct FontBLF *font,
                              const char *str,
-                             size_t len,
+                             size_t str_len,
                              struct rctf *r_box,
                              struct ResultBLF *r_info);
 void blf_font_width_and_height(struct FontBLF *font,
                                const char *str,
-                               size_t len,
+                               size_t str_len,
                                float *r_width,
                                float *r_height,
                                struct ResultBLF *r_info);
-float blf_font_width(struct FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info);
-float blf_font_height(struct FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info);
+float blf_font_width(struct FontBLF *font,
+                     const char *str,
+                     size_t str_len,
+                     struct ResultBLF *r_info);
+float blf_font_height(struct FontBLF *font,
+                      const char *str,
+                      size_t str_len,
+                      struct ResultBLF *r_info);
 float blf_font_fixed_width(struct FontBLF *font);
 int blf_font_height_max(struct FontBLF *font);
 int blf_font_width_max(struct FontBLF *font);
@@ -103,7 +112,7 @@ char *blf_display_name(struct FontBLF *font);
 
 void blf_font_boundbox_foreach_glyph(struct FontBLF *font,
                                      const char *str,
-                                     size_t len,
+                                     size_t str_len,
                                      bool (*user_fn)(const char *str,
                                                      const size_t str_step_ofs,
                                                      const struct rcti *glyph_step_bounds,
@@ -116,15 +125,11 @@ void blf_font_boundbox_foreach_glyph(struct FontBLF *font,
 
 int blf_font_count_missing_chars(struct FontBLF *font,
                                  const char *str,
-                                 const size_t len,
+                                 const size_t str_len,
                                  int *r_tot_chars);
 
 void blf_font_free(struct FontBLF *font);
 
-struct KerningCacheBLF *blf_kerning_cache_find(struct FontBLF *font);
-struct KerningCacheBLF *blf_kerning_cache_new(struct FontBLF *font, struct GlyphCacheBLF *gc);
-void blf_kerning_cache_clear(struct FontBLF *font);
-
 struct GlyphCacheBLF *blf_glyph_cache_find(struct FontBLF *font,
                                            unsigned int size,
                                            unsigned int dpi);

source/blender/blenfont/intern/blf_internal_types.h

@@ -34,6 +34,9 @@
 /* Number of characters in KerningCacheBLF.table. */
 #define KERNING_CACHE_TABLE_SIZE 128
 
+/* A value in the kerning cache that indicates it is not yet set. */
+#define KERNING_ENTRY_UNSET INT_MAX
+
 typedef struct BatchBLF {
   struct FontBLF *font; /* can only batch glyph from the same font */
   struct GPUBatch *batch;
@@ -50,7 +53,6 @@ typedef struct BatchBLF {
 extern BatchBLF g_batch;
 
 typedef struct KerningCacheBLF {
-  struct KerningCacheBLF *next, *prev;
   /**
    * Cache a ascii glyph pairs. Only store the x offset we are interested in,
    * instead of the full #FT_Vector since it's not used for drawing at the moment.
@@ -223,10 +225,7 @@ typedef struct FontBLF {
    */
   ListBase cache;
 
-  /* list of kerning cache for this font. */
-  ListBase kerning_caches;
-
-  /* current kerning cache for this font and kerning mode. */
+  /* Cache of unscaled kerning values. Will be NULL if font does not have kerning. */
   KerningCacheBLF *kerning_cache;
 
   /* freetype2 lib handle. */

source/blender/blenkernel/BKE_anonymous_attribute.h

@@ -0,0 +1,37 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct AnonymousAttributeID AnonymousAttributeID;
+
+AnonymousAttributeID *BKE_anonymous_attribute_id_new_weak(const char *debug_name);
+AnonymousAttributeID *BKE_anonymous_attribute_id_new_strong(const char *debug_name);
+bool BKE_anonymous_attribute_id_has_strong_references(const AnonymousAttributeID *anonymous_id);
+void BKE_anonymous_attribute_id_increment_weak(const AnonymousAttributeID *anonymous_id);
+void BKE_anonymous_attribute_id_increment_strong(const AnonymousAttributeID *anonymous_id);
+void BKE_anonymous_attribute_id_decrement_weak(const AnonymousAttributeID *anonymous_id);
+void BKE_anonymous_attribute_id_decrement_strong(const AnonymousAttributeID *anonymous_id);
+const char *BKE_anonymous_attribute_id_debug_name(const AnonymousAttributeID *anonymous_id);
+const char *BKE_anonymous_attribute_id_internal_name(const AnonymousAttributeID *anonymous_id);
+
+#ifdef __cplusplus
+}
+#endif

source/blender/blenkernel/BKE_anonymous_attribute.hh

@@ -0,0 +1,223 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#pragma once
+
+#include <atomic>
+#include <string>
+
+#include "BLI_hash.hh"
+#include "BLI_string_ref.hh"
+
+#include "BKE_anonymous_attribute.h"
+
+namespace blender::bke {
+
+template<bool IsStrongReference> class OwnedAnonymousAttributeID {
+ private:
+  const AnonymousAttributeID *data_ = nullptr;
+
+ public:
+  OwnedAnonymousAttributeID() = default;
+
+  explicit OwnedAnonymousAttributeID(StringRefNull debug_name)
+  {
+    if constexpr (IsStrongReference) {
+      data_ = BKE_anonymous_attribute_id_new_strong(debug_name.c_str());
+    }
+    else {
+      data_ = BKE_anonymous_attribute_id_new_weak(debug_name.c_str());
+    }
+  }
+
+  /* This transfers ownership, so no incref is necessary. */
+  explicit OwnedAnonymousAttributeID(const AnonymousAttributeID *anonymous_id)
+      : data_(anonymous_id)
+  {
+  }
+
+  template<bool OtherIsStrong>
+  OwnedAnonymousAttributeID(const OwnedAnonymousAttributeID<OtherIsStrong> &other)
+  {
+    data_ = other.data_;
+    this->incref();
+  }
+
+  template<bool OtherIsStrong>
+  OwnedAnonymousAttributeID(OwnedAnonymousAttributeID<OtherIsStrong> &&other)
+  {
+    data_ = other.data_;
+    this->incref();
+    other.decref();
+    other.data_ = nullptr;
+  }
+
+  ~OwnedAnonymousAttributeID()
+  {
+    this->decref();
+  }
+
+  template<bool OtherIsStrong>
+  OwnedAnonymousAttributeID &operator=(const OwnedAnonymousAttributeID<OtherIsStrong> &other)
+  {
+    if (this == &other) {
+      return *this;
+    }
+    this->~OwnedAnonymousAttributeID();
+    new (this) OwnedAnonymousAttributeID(other);
+    return *this;
+  }
+
+  template<bool OtherIsStrong>
+  OwnedAnonymousAttributeID &operator=(OwnedAnonymousAttributeID<OtherIsStrong> &&other)
+  {
+    if (this == &other) {
+      return *this;
+    }
+    this->~OwnedAnonymousAttributeID();
+    new (this) OwnedAnonymousAttributeID(std::move(other));
+    return *this;
+  }
+
+  operator bool() const
+  {
+    return data_ != nullptr;
+  }
+
+  StringRefNull debug_name() const
+  {
+    BLI_assert(data_ != nullptr);
+    return BKE_anonymous_attribute_id_debug_name(data_);
+  }
+
+  bool has_strong_references() const
+  {
+    BLI_assert(data_ != nullptr);
+    return BKE_anonymous_attribute_id_has_strong_references(data_);
+  }
+
+  const AnonymousAttributeID *extract()
+  {
+    const AnonymousAttributeID *extracted_data = data_;
+    /* Don't decref because the caller becomes the new owner. */
+    data_ = nullptr;
+    return extracted_data;
+  }
+
+  const AnonymousAttributeID *get()
+  {
+    return data_;
+  }
+
+ private:
+  void incref()
+  {
+    if (data_ == nullptr) {
+      return;
+    }
+    if constexpr (IsStrongReference) {
+      BKE_anonymous_attribute_id_increment_strong(data_);
+    }
+    else {
+      BKE_anonymous_attribute_id_increment_weak(data_);
+    }
+  }
+
+  void decref()
+  {
+    if (data_ == nullptr) {
+      return;
+    }
+    if constexpr (IsStrongReference) {
+      BKE_anonymous_attribute_id_decrement_strong(data_);
+    }
+    else {
+      BKE_anonymous_attribute_id_decrement_weak(data_);
+    }
+  }
+};
+
+using StrongAnonymousAttributeID = OwnedAnonymousAttributeID<true>;
+using WeakAnonymousAttributeID = OwnedAnonymousAttributeID<false>;
+
+class AttributeIDRef {
+ private:
+  StringRef name_;
+  const AnonymousAttributeID *anonymous_id_ = nullptr;
+
+ public:
+  AttributeIDRef() = default;
+
+  AttributeIDRef(StringRef name) : name_(name)
+  {
+  }
+
+  AttributeIDRef(StringRefNull name) : name_(name)
+  {
+  }
+
+  AttributeIDRef(const char *name) : name_(name)
+  {
+  }
+
+  AttributeIDRef(const std::string &name) : name_(name)
+  {
+  }
+
+  /* The anonymous id is only borrowed, the caller has to keep a reference to it. */
+  AttributeIDRef(const AnonymousAttributeID *anonymous_id) : anonymous_id_(anonymous_id)
+  {
+  }
+
+  operator bool() const
+  {
+    return this->is_named() || this->is_anonymous();
+  }
+
+  friend bool operator==(const AttributeIDRef &a, const AttributeIDRef &b)
+  {
+    return a.anonymous_id_ == b.anonymous_id_ && a.name_ == b.name_;
+  }
+
+  uint64_t hash() const
+  {
+    return get_default_hash_2(name_, anonymous_id_);
+  }
+
+  bool is_named() const
+  {
+    return !name_.is_empty();
+  }
+
+  bool is_anonymous() const
+  {
+    return anonymous_id_ != nullptr;
+  }
+
+  StringRef name() const
+  {
+    BLI_assert(this->is_named());
+    return name_;
+  }
+
+  const AnonymousAttributeID &anonymous_id() const
+  {
+    BLI_assert(this->is_anonymous());
+    return *anonymous_id_;
+  }
+};
+
+}  // namespace blender::bke

source/blender/blenkernel/BKE_attribute_access.hh

@@ -22,6 +22,7 @@
 #include "FN_generic_span.hh"
 #include "FN_generic_virtual_array.hh"
 
+#include "BKE_anonymous_attribute.hh"
 #include "BKE_attribute.h"
 
 #include "BLI_color.hh"
@@ -104,8 +105,8 @@ struct AttributeInitMove : public AttributeInit {
 };
 
 /* Returns false when the iteration should be stopped. */
-using AttributeForeachCallback = blender::FunctionRef<bool(blender::StringRefNull attribute_name,
-                                                           const AttributeMetaData &meta_data)>;
+using AttributeForeachCallback = blender::FunctionRef<bool(
+    const blender::bke::AttributeIDRef &attribute_id, const AttributeMetaData &meta_data)>;
 
 namespace blender::bke {
 
@@ -333,26 +334,30 @@ class CustomDataAttributes {
 
   void reallocate(const int size);
 
-  std::optional<blender::fn::GSpan> get_for_read(const blender::StringRef name) const;
+  std::optional<blender::fn::GSpan> get_for_read(
+      const blender::bke::AttributeIDRef &attribute_id) const;
 
-  blender::fn::GVArrayPtr get_for_read(const StringRef name,
+  blender::fn::GVArrayPtr get_for_read(const AttributeIDRef &attribute_id,
                                        const CustomDataType data_type,
                                        const void *default_value) const;
 
   template<typename T>
-  blender::fn::GVArray_Typed<T> get_for_read(const blender::StringRef name,
+  blender::fn::GVArray_Typed<T> get_for_read(const blender::bke::AttributeIDRef &attribute_id,
                                              const T &default_value) const
   {
     const blender::fn::CPPType &cpp_type = blender::fn::CPPType::get<T>();
     const CustomDataType type = blender::bke::cpp_type_to_custom_data_type(cpp_type);
-    GVArrayPtr varray = this->get_for_read(name, type, &default_value);
+    GVArrayPtr varray = this->get_for_read(attribute_id, type, &default_value);
     return blender::fn::GVArray_Typed<T>(std::move(varray));
   }
 
-  std::optional<blender::fn::GMutableSpan> get_for_write(const blender::StringRef name);
-  bool create(const blender::StringRef name, const CustomDataType data_type);
-  bool create_by_move(const blender::StringRef name, const CustomDataType data_type, void *buffer);
-  bool remove(const blender::StringRef name);
+  std::optional<blender::fn::GMutableSpan> get_for_write(
+      const blender::bke::AttributeIDRef &attribute_id);
+  bool create(const blender::bke::AttributeIDRef &attribute_id, const CustomDataType data_type);
+  bool create_by_move(const blender::bke::AttributeIDRef &attribute_id,
+                      const CustomDataType data_type,
+                      void *buffer);
+  bool remove(const blender::bke::AttributeIDRef &attribute_id);
 
   bool foreach_attribute(const AttributeForeachCallback callback,
                          const AttributeDomain domain) const;

source/blender/blenkernel/BKE_cachefile.h

@@ -61,11 +61,6 @@ void BKE_cachefile_reader_open(struct CacheFile *cache_file,
                                const char *object_path);
 void BKE_cachefile_reader_free(struct CacheFile *cache_file, struct CacheReader **reader);
 
-/* Determine whether the CacheFile should use a render engine procedural. If so, data is not read
- * from the file and bouding boxes are used to represent the objects in the Scene. Render engines
- * will receive the bounding box as a placeholder but can instead load the data directly if they
- * support it.
- */
 bool BKE_cache_file_uses_render_procedural(const struct CacheFile *cache_file,
                                            struct Scene *scene,
                                            const int dag_eval_mode);

source/blender/blenkernel/BKE_customdata.h

@@ -33,6 +33,7 @@
 extern "C" {
 #endif
 
+struct AnonymousAttributeID;
 struct BMesh;
 struct BlendDataReader;
 struct BlendWriter;
@@ -193,6 +194,12 @@ void *CustomData_add_layer_named(struct CustomData *data,
                                  void *layer,
                                  int totelem,
                                  const char *name);
+void *CustomData_add_layer_anonymous(struct CustomData *data,
+                                     int type,
+                                     eCDAllocType alloctype,
+                                     void *layer,
+                                     int totelem,
+                                     const struct AnonymousAttributeID *anonymous_id);
 
 /* frees the active or first data layer with the give type.
  * returns 1 on success, 0 if no layer with the given type is found
@@ -231,6 +238,11 @@ void *CustomData_duplicate_referenced_layer_named(struct CustomData *data,
                                                   const int type,
                                                   const char *name,
                                                   const int totelem);
+void *CustomData_duplicate_referenced_layer_anonymous(
+    CustomData *data,
+    const int type,
+    const struct AnonymousAttributeID *anonymous_id,
+    const int totelem);
 bool CustomData_is_referenced_layer(struct CustomData *data, int type);
 
 /* Duplicate all the layers with flag NOFREE, and remove the flag from duplicated layers. */

source/blender/blenkernel/BKE_geometry_set.hh

@@ -31,6 +31,7 @@
 #include "BLI_user_counter.hh"
 #include "BLI_vector_set.hh"
 
+#include "BKE_anonymous_attribute.hh"
 #include "BKE_attribute_access.hh"
 #include "BKE_geometry_set.h"
 
@@ -88,11 +89,11 @@ class GeometryComponent {
   GeometryComponentType type() const;
 
   /* Return true when any attribute with this name exists, including built in attributes. */
-  bool attribute_exists(const blender::StringRef attribute_name) const;
+  bool attribute_exists(const blender::bke::AttributeIDRef &attribute_id) const;
 
   /* Return the data type and domain of an attribute with the given name if it exists. */
   std::optional<AttributeMetaData> attribute_get_meta_data(
-      const blender::StringRef attribute_name) const;
+      const blender::bke::AttributeIDRef &attribute_id) const;
 
   /* Returns true when the geometry component supports this attribute domain. */
   bool attribute_domain_supported(const AttributeDomain domain) const;
@@ -104,12 +105,12 @@ class GeometryComponent {
   /* Get read-only access to the highest priority attribute with the given name.
    * Returns null if the attribute does not exist. */
   blender::bke::ReadAttributeLookup attribute_try_get_for_read(
-      const blender::StringRef attribute_name) const;
+      const blender::bke::AttributeIDRef &attribute_id) const;
 
   /* Get read and write access to the highest priority attribute with the given name.
    * Returns null if the attribute does not exist. */
   blender::bke::WriteAttributeLookup attribute_try_get_for_write(
-      const blender::StringRef attribute_name);
+      const blender::bke::AttributeIDRef &attribute_id);
 
   /* Get a read-only attribute for the domain based on the given attribute. This can be used to
    * interpolate from one domain to another.
@@ -120,10 +121,10 @@ class GeometryComponent {
       const AttributeDomain to_domain) const;
 
   /* Returns true when the attribute has been deleted. */
-  bool attribute_try_delete(const blender::StringRef attribute_name);
+  bool attribute_try_delete(const blender::bke::AttributeIDRef &attribute_id);
 
   /* Returns true when the attribute has been created. */
-  bool attribute_try_create(const blender::StringRef attribute_name,
+  bool attribute_try_create(const blender::bke::AttributeIDRef &attribute_id,
                             const AttributeDomain domain,
                             const CustomDataType data_type,
                             const AttributeInit &initializer);
@@ -133,7 +134,7 @@ class GeometryComponent {
   bool attribute_try_create_builtin(const blender::StringRef attribute_name,
                                     const AttributeInit &initializer);
 
-  blender::Set<std::string> attribute_names() const;
+  blender::Set<blender::bke::AttributeIDRef> attribute_ids() const;
   bool attribute_foreach(const AttributeForeachCallback callback) const;
 
   virtual bool is_empty() const;
@@ -142,7 +143,7 @@ class GeometryComponent {
    * Returns null when the attribute does not exist or cannot be converted to the requested domain
    * and data type. */
   std::unique_ptr<blender::fn::GVArray> attribute_try_get_for_read(
-      const blender::StringRef attribute_name,
+      const blender::bke::AttributeIDRef &attribute_id,
       const AttributeDomain domain,
       const CustomDataType data_type) const;
 
@@ -150,18 +151,18 @@ class GeometryComponent {
    * left unchanged. Returns null when the attribute does not exist or cannot be adapted to the
    * requested domain. */
   std::unique_ptr<blender::fn::GVArray> attribute_try_get_for_read(
-      const blender::StringRef attribute_name, const AttributeDomain domain) const;
+      const blender::bke::AttributeIDRef &attribute_id, const AttributeDomain domain) const;
 
   /* Get a virtual array to read data of an attribute with the given data type. The domain is
    * left unchanged. Returns null when the attribute does not exist or cannot be converted to the
    * requested data type. */
   blender::bke::ReadAttributeLookup attribute_try_get_for_read(
-      const blender::StringRef attribute_name, const CustomDataType data_type) const;
+      const blender::bke::AttributeIDRef &attribute_id, const CustomDataType data_type) const;
 
   /* Get a virtual array to read the data of an attribute. If that is not possible, the returned
    * virtual array will contain a default value. This never returns null. */
   std::unique_ptr<blender::fn::GVArray> attribute_get_for_read(
-      const blender::StringRef attribute_name,
+      const blender::bke::AttributeIDRef &attribute_id,
       const AttributeDomain domain,
       const CustomDataType data_type,
       const void *default_value = nullptr) const;
@@ -169,14 +170,15 @@ class GeometryComponent {
   /* Should be used instead of the method above when the requested data type is known at compile
    * time for better type safety. */
   template<typename T>
-  blender::fn::GVArray_Typed<T> attribute_get_for_read(const blender::StringRef attribute_name,
-                                                       const AttributeDomain domain,
-                                                       const T &default_value) const
+  blender::fn::GVArray_Typed<T> attribute_get_for_read(
+      const blender::bke::AttributeIDRef &attribute_id,
+      const AttributeDomain domain,
+      const T &default_value) const
   {
     const blender::fn::CPPType &cpp_type = blender::fn::CPPType::get<T>();
     const CustomDataType type = blender::bke::cpp_type_to_custom_data_type(cpp_type);
     std::unique_ptr varray = this->attribute_get_for_read(
-        attribute_name, domain, type, &default_value);
+        attribute_id, domain, type, &default_value);
     return blender::fn::GVArray_Typed<T>(std::move(varray));
   }
 
@@ -191,7 +193,7 @@ class GeometryComponent {
    *   is created that will overwrite the existing attribute in the end.
    */
   blender::bke::OutputAttribute attribute_try_get_for_output(
-      const blender::StringRef attribute_name,
+      const blender::bke::AttributeIDRef &attribute_id,
       const AttributeDomain domain,
       const CustomDataType data_type,
       const void *default_value = nullptr);
@@ -200,28 +202,30 @@ class GeometryComponent {
    * attributes are not read, i.e. the attribute is used only for output. Since values are not read
    * from this attribute, no default value is necessary. */
   blender::bke::OutputAttribute attribute_try_get_for_output_only(
-      const blender::StringRef attribute_name,
+      const blender::bke::AttributeIDRef &attribute_id,
       const AttributeDomain domain,
       const CustomDataType data_type);
 
   /* Statically typed method corresponding to the equally named generic one. */
   template<typename T>
   blender::bke::OutputAttribute_Typed<T> attribute_try_get_for_output(
-      const blender::StringRef attribute_name, const AttributeDomain domain, const T default_value)
+      const blender::bke::AttributeIDRef &attribute_id,
+      const AttributeDomain domain,
+      const T default_value)
   {
     const blender::fn::CPPType &cpp_type = blender::fn::CPPType::get<T>();
     const CustomDataType data_type = blender::bke::cpp_type_to_custom_data_type(cpp_type);
-    return this->attribute_try_get_for_output(attribute_name, domain, data_type, &default_value);
+    return this->attribute_try_get_for_output(attribute_id, domain, data_type, &default_value);
   }
 
   /* Statically typed method corresponding to the equally named generic one. */
   template<typename T>
   blender::bke::OutputAttribute_Typed<T> attribute_try_get_for_output_only(
-      const blender::StringRef attribute_name, const AttributeDomain domain)
+      const blender::bke::AttributeIDRef &attribute_id, const AttributeDomain domain)
   {
     const blender::fn::CPPType &cpp_type = blender::fn::CPPType::get<T>();
     const CustomDataType data_type = blender::bke::cpp_type_to_custom_data_type(cpp_type);
-    return this->attribute_try_get_for_output_only(attribute_name, domain, data_type);
+    return this->attribute_try_get_for_output_only(attribute_id, domain, data_type);
   }
 
  private:

source/blender/blenkernel/BKE_geometry_set_instances.hh

@@ -59,9 +59,10 @@ struct AttributeKind {
  * will contain the highest complexity data type and the highest priority domain among every
  * attribute with the given name on all of the input components.
  */
-void geometry_set_gather_instances_attribute_info(Span<GeometryInstanceGroup> set_groups,
-                                                  Span<GeometryComponentType> component_types,
-                                                  const Set<std::string> &ignored_attributes,
-                                                  Map<std::string, AttributeKind> &r_attributes);
+void geometry_set_gather_instances_attribute_info(
+    Span<GeometryInstanceGroup> set_groups,
+    Span<GeometryComponentType> component_types,
+    const Set<std::string> &ignored_attributes,
+    Map<AttributeIDRef, AttributeKind> &r_attributes);
 
 }  // namespace blender::bke

source/blender/blenkernel/BKE_node.h

@@ -111,8 +111,7 @@ typedef struct bNodeSocketTemplate {
 #ifdef __cplusplus
 namespace blender {
 namespace nodes {
-class SocketMFNetworkBuilder;
-class NodeMFNetworkBuilder;
+class NodeMultiFunctionBuilder;
 class GeoNodeExecParams;
 }  // namespace nodes
 namespace fn {
@@ -121,18 +120,16 @@ class MFDataType;
 }  // namespace fn
 }  // namespace blender
 
-using NodeExpandInMFNetworkFunction = void (*)(blender::nodes::NodeMFNetworkBuilder &builder);
+using NodeMultiFunctionBuildFunction = void (*)(blender::nodes::NodeMultiFunctionBuilder &builder);
 using NodeGeometryExecFunction = void (*)(blender::nodes::GeoNodeExecParams params);
 using SocketGetCPPTypeFunction = const blender::fn::CPPType *(*)();
 using SocketGetCPPValueFunction = void (*)(const struct bNodeSocket &socket, void *r_value);
 using SocketGetGeometryNodesCPPTypeFunction = const blender::fn::CPPType *(*)();
 using SocketGetGeometryNodesCPPValueFunction = void (*)(const struct bNodeSocket &socket,
                                                         void *r_value);
-using SocketExpandInMFNetworkFunction = void (*)(blender::nodes::SocketMFNetworkBuilder &builder);
 
 #else
-typedef void *NodeExpandInMFNetworkFunction;
-typedef void *SocketExpandInMFNetworkFunction;
+typedef void *NodeMultiFunctionBuildFunction;
 typedef void *NodeGeometryExecFunction;
 typedef void *SocketGetCPPTypeFunction;
 typedef void *SocketGetGeometryNodesCPPTypeFunction;
@@ -196,8 +193,6 @@ typedef struct bNodeSocketType {
   /* Callback to free the socket type. */
   void (*free_self)(struct bNodeSocketType *stype);
 
-  /* Expands the socket into a multi-function node that outputs the socket value. */
-  SocketExpandInMFNetworkFunction expand_in_mf_network;
   /* Return the CPPType of this socket. */
   SocketGetCPPTypeFunction get_base_cpp_type;
   /* Get the value of this socket in a generic way. */
@@ -332,8 +327,8 @@ typedef struct bNodeType {
   /* gpu */
   NodeGPUExecFunction gpu_fn;
 
-  /* Expands the bNode into nodes in a multi-function network, which will be evaluated later on. */
-  NodeExpandInMFNetworkFunction expand_in_mf_network;
+  /* Build a multi-function for this node. */
+  NodeMultiFunctionBuildFunction build_multi_function;
 
   /* Execute a geometry node. */
   NodeGeometryExecFunction geometry_node_execute;
@@ -351,7 +346,7 @@ typedef struct bNodeType {
 #define NODE_CLASS_OP_FILTER 5
 #define NODE_CLASS_GROUP 6
 // #define NODE_CLASS_FILE              7
-#define NODE_CLASS_CONVERTOR 8
+#define NODE_CLASS_CONVERTER 8
 #define NODE_CLASS_MATTE 9
 #define NODE_CLASS_DISTORT 10
 // #define NODE_CLASS_OP_DYNAMIC        11 /* deprecated */

source/blender/blenkernel/CMakeLists.txt

@@ -76,6 +76,7 @@ set(SRC
   intern/anim_path.c
   intern/anim_sys.c
   intern/anim_visualization.c
+  intern/anonymous_attribute.cc
   intern/appdir.c
   intern/armature.c
   intern/armature_selection.cc
@@ -295,6 +296,8 @@ set(SRC
   BKE_anim_path.h
   BKE_anim_visualization.h
   BKE_animsys.h
+  BKE_anonymous_attribute.h
+  BKE_anonymous_attribute.hh
   BKE_appdir.h
   BKE_armature.h
   BKE_armature.hh

source/blender/blenkernel/intern/anonymous_attribute.cc

@@ -0,0 +1,92 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include "BKE_anonymous_attribute.hh"
+
+using namespace blender::bke;
+
+struct AnonymousAttributeID {
+  mutable std::atomic<int> refcount_weak = 0;
+  mutable std::atomic<int> refcount_strong = 0;
+  std::string debug_name;
+  std::string internal_name;
+};
+
+static std::string get_new_internal_name()
+{
+  static std::atomic<int> index = 0;
+  const int next_index = index.fetch_add(1);
+  return "anonymous_attribute_" + std::to_string(next_index);
+}
+
+AnonymousAttributeID *BKE_anonymous_attribute_id_new_weak(const char *debug_name)
+{
+  AnonymousAttributeID *anonymous_id = new AnonymousAttributeID();
+  anonymous_id->debug_name = debug_name;
+  anonymous_id->internal_name = get_new_internal_name();
+  anonymous_id->refcount_weak.store(1);
+  return anonymous_id;
+}
+
+AnonymousAttributeID *BKE_anonymous_attribute_id_new_strong(const char *debug_name)
+{
+  AnonymousAttributeID *anonymous_id = new AnonymousAttributeID();
+  anonymous_id->debug_name = debug_name;
+  anonymous_id->internal_name = get_new_internal_name();
+  anonymous_id->refcount_weak.store(1);
+  anonymous_id->refcount_strong.store(1);
+  return anonymous_id;
+}
+
+bool BKE_anonymous_attribute_id_has_strong_references(const AnonymousAttributeID *anonymous_id)
+{
+  return anonymous_id->refcount_strong.load() >= 1;
+}
+
+void BKE_anonymous_attribute_id_increment_weak(const AnonymousAttributeID *anonymous_id)
+{
+  anonymous_id->refcount_weak.fetch_add(1);
+}
+
+void BKE_anonymous_attribute_id_increment_strong(const AnonymousAttributeID *anonymous_id)
+{
+  anonymous_id->refcount_weak.fetch_add(1);
+  anonymous_id->refcount_strong.fetch_add(1);
+}
+
+void BKE_anonymous_attribute_id_decrement_weak(const AnonymousAttributeID *anonymous_id)
+{
+  const int new_refcount = anonymous_id->refcount_weak.fetch_sub(1) - 1;
+  if (new_refcount == 0) {
+    delete anonymous_id;
+  }
+}
+
+void BKE_anonymous_attribute_id_decrement_strong(const AnonymousAttributeID *anonymous_id)
+{
+  anonymous_id->refcount_strong.fetch_sub(1);
+  BKE_anonymous_attribute_id_decrement_weak(anonymous_id);
+}
+
+const char *BKE_anonymous_attribute_id_debug_name(const AnonymousAttributeID *anonymous_id)
+{
+  return anonymous_id->debug_name.c_str();
+}
+
+const char *BKE_anonymous_attribute_id_internal_name(const AnonymousAttributeID *anonymous_id)
+{
+  return anonymous_id->internal_name.c_str();
+}

source/blender/blenkernel/intern/attribute_access.cc

@@ -334,8 +334,20 @@ bool BuiltinCustomDataLayerProvider::exists(const GeometryComponent &component)
   return data != nullptr;
 }
 
+static bool custom_data_layer_matches_attribute_id(const CustomDataLayer &layer,
+                                                   const AttributeIDRef &attribute_id)
+{
+  if (!attribute_id) {
+    return false;
+  }
+  if (attribute_id.is_anonymous()) {
+    return layer.anonymous_id == &attribute_id.anonymous_id();
+  }
+  return layer.name == attribute_id.name();
+}
+
 ReadAttributeLookup CustomDataAttributeProvider::try_get_for_read(
-    const GeometryComponent &component, const StringRef attribute_name) const
+    const GeometryComponent &component, const AttributeIDRef &attribute_id) const
 {
   const CustomData *custom_data = custom_data_access_.get_const_custom_data(component);
   if (custom_data == nullptr) {
@@ -343,7 +355,7 @@ ReadAttributeLookup CustomDataAttributeProvider::try_get_for_read(
   }
   const int domain_size = component.attribute_domain_size(domain_);
   for (const CustomDataLayer &layer : Span(custom_data->layers, custom_data->totlayer)) {
-    if (layer.name != attribute_name) {
+    if (!custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       continue;
     }
     const CustomDataType data_type = (CustomDataType)layer.type;
@@ -368,7 +380,7 @@ ReadAttributeLookup CustomDataAttributeProvider::try_get_for_read(
 }
 
 WriteAttributeLookup CustomDataAttributeProvider::try_get_for_write(
-    GeometryComponent &component, const StringRef attribute_name) const
+    GeometryComponent &component, const AttributeIDRef &attribute_id) const
 {
   CustomData *custom_data = custom_data_access_.get_custom_data(component);
   if (custom_data == nullptr) {
@@ -376,10 +388,17 @@ WriteAttributeLookup CustomDataAttributeProvider::try_get_for_write(
   }
   const int domain_size = component.attribute_domain_size(domain_);
   for (CustomDataLayer &layer : MutableSpan(custom_data->layers, custom_data->totlayer)) {
-    if (layer.name != attribute_name) {
+    if (!custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       continue;
     }
-    CustomData_duplicate_referenced_layer_named(custom_data, layer.type, layer.name, domain_size);
+    if (attribute_id.is_named()) {
+      CustomData_duplicate_referenced_layer_named(
+          custom_data, layer.type, layer.name, domain_size);
+    }
+    else {
+      CustomData_duplicate_referenced_layer_anonymous(
+          custom_data, layer.type, &attribute_id.anonymous_id(), domain_size);
+    }
     const CustomDataType data_type = (CustomDataType)layer.type;
     switch (data_type) {
       case CD_PROP_FLOAT:
@@ -402,7 +421,7 @@ WriteAttributeLookup CustomDataAttributeProvider::try_get_for_write(
 }
 
 bool CustomDataAttributeProvider::try_delete(GeometryComponent &component,
-                                             const StringRef attribute_name) const
+                                             const AttributeIDRef &attribute_id) const
 {
   CustomData *custom_data = custom_data_access_.get_custom_data(component);
   if (custom_data == nullptr) {
@@ -411,7 +430,8 @@ bool CustomDataAttributeProvider::try_delete(GeometryComponent &component,
   const int domain_size = component.attribute_domain_size(domain_);
   for (const int i : IndexRange(custom_data->totlayer)) {
     const CustomDataLayer &layer = custom_data->layers[i];
-    if (this->type_is_supported((CustomDataType)layer.type) && layer.name == attribute_name) {
+    if (this->type_is_supported((CustomDataType)layer.type) &&
+        custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       CustomData_free_layer(custom_data, layer.type, domain_size, i);
       return true;
     }
@@ -419,24 +439,39 @@ bool CustomDataAttributeProvider::try_delete(GeometryComponent &component,
   return false;
 }
 
-static bool add_named_custom_data_layer_from_attribute_init(const StringRef attribute_name,
-                                                            CustomData &custom_data,
-                                                            const CustomDataType data_type,
-                                                            const int domain_size,
-                                                            const AttributeInit &initializer)
+static void *add_generic_custom_data_layer(CustomData &custom_data,
+                                           const CustomDataType data_type,
+                                           const eCDAllocType alloctype,
+                                           void *layer_data,
+                                           const int domain_size,
+                                           const AttributeIDRef &attribute_id)
 {
-  char attribute_name_c[MAX_NAME];
-  attribute_name.copy(attribute_name_c);
+  if (attribute_id.is_named()) {
+    char attribute_name_c[MAX_NAME];
+    attribute_id.name().copy(attribute_name_c);
+    return CustomData_add_layer_named(
+        &custom_data, data_type, CD_DEFAULT, nullptr, domain_size, attribute_name_c);
+  }
+  const AnonymousAttributeID &anonymous_id = attribute_id.anonymous_id();
+  return CustomData_add_layer_anonymous(
+      &custom_data, data_type, alloctype, layer_data, domain_size, &anonymous_id);
+}
 
+static bool add_custom_data_layer_from_attribute_init(const AttributeIDRef &attribute_id,
+                                                      CustomData &custom_data,
+                                                      const CustomDataType data_type,
+                                                      const int domain_size,
+                                                      const AttributeInit &initializer)
+{
   switch (initializer.type) {
     case AttributeInit::Type::Default: {
-      void *data = CustomData_add_layer_named(
-          &custom_data, data_type, CD_DEFAULT, nullptr, domain_size, attribute_name_c);
+      void *data = add_generic_custom_data_layer(
+          custom_data, data_type, CD_DEFAULT, nullptr, domain_size, attribute_id);
       return data != nullptr;
     }
     case AttributeInit::Type::VArray: {
-      void *data = CustomData_add_layer_named(
-          &custom_data, data_type, CD_DEFAULT, nullptr, domain_size, attribute_name_c);
+      void *data = add_generic_custom_data_layer(
+          custom_data, data_type, CD_DEFAULT, nullptr, domain_size, attribute_id);
       if (data == nullptr) {
         return false;
       }
@@ -446,8 +481,8 @@ static bool add_named_custom_data_layer_from_attribute_init(const StringRef attr
     }
     case AttributeInit::Type::MoveArray: {
       void *source_data = static_cast<const AttributeInitMove &>(initializer).data;
-      void *data = CustomData_add_layer_named(
-          &custom_data, data_type, CD_ASSIGN, source_data, domain_size, attribute_name_c);
+      void *data = add_generic_custom_data_layer(
+          custom_data, data_type, CD_ASSIGN, source_data, domain_size, attribute_id);
       if (data == nullptr) {
         MEM_freeN(source_data);
         return false;
@@ -461,7 +496,7 @@ static bool add_named_custom_data_layer_from_attribute_init(const StringRef attr
 }
 
 bool CustomDataAttributeProvider::try_create(GeometryComponent &component,
-                                             const StringRef attribute_name,
+                                             const AttributeIDRef &attribute_id,
                                              const AttributeDomain domain,
                                              const CustomDataType data_type,
                                              const AttributeInit &initializer) const
@@ -477,13 +512,13 @@ bool CustomDataAttributeProvider::try_create(GeometryComponent &component,
     return false;
   }
   for (const CustomDataLayer &layer : Span(custom_data->layers, custom_data->totlayer)) {
-    if (layer.name == attribute_name) {
+    if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       return false;
     }
   }
   const int domain_size = component.attribute_domain_size(domain_);
-  add_named_custom_data_layer_from_attribute_init(
-      attribute_name, *custom_data, data_type, domain_size, initializer);
+  add_custom_data_layer_from_attribute_init(
+      attribute_id, *custom_data, data_type, domain_size, initializer);
   return true;
 }
 
@@ -498,7 +533,14 @@ bool CustomDataAttributeProvider::foreach_attribute(const GeometryComponent &com
     const CustomDataType data_type = (CustomDataType)layer.type;
     if (this->type_is_supported(data_type)) {
       AttributeMetaData meta_data{domain_, data_type};
-      if (!callback(layer.name, meta_data)) {
+      AttributeIDRef attribute_id;
+      if (layer.anonymous_id != nullptr) {
+        attribute_id = layer.anonymous_id;
+      }
+      else {
+        attribute_id = layer.name;
+      }
+      if (!callback(attribute_id, meta_data)) {
         return false;
       }
     }
@@ -507,7 +549,7 @@ bool CustomDataAttributeProvider::foreach_attribute(const GeometryComponent &com
 }
 
 ReadAttributeLookup NamedLegacyCustomDataProvider::try_get_for_read(
-    const GeometryComponent &component, const StringRef attribute_name) const
+    const GeometryComponent &component, const AttributeIDRef &attribute_id) const
 {
   const CustomData *custom_data = custom_data_access_.get_const_custom_data(component);
   if (custom_data == nullptr) {
@@ -515,7 +557,7 @@ ReadAttributeLookup NamedLegacyCustomDataProvider::try_get_for_read(
   }
   for (const CustomDataLayer &layer : Span(custom_data->layers, custom_data->totlayer)) {
     if (layer.type == stored_type_) {
-      if (layer.name == attribute_name) {
+      if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
         const int domain_size = component.attribute_domain_size(domain_);
         return {as_read_attribute_(layer.data, domain_size), domain_};
       }
@@ -525,7 +567,7 @@ ReadAttributeLookup NamedLegacyCustomDataProvider::try_get_for_read(
 }
 
 WriteAttributeLookup NamedLegacyCustomDataProvider::try_get_for_write(
-    GeometryComponent &component, const StringRef attribute_name) const
+    GeometryComponent &component, const AttributeIDRef &attribute_id) const
 {
   CustomData *custom_data = custom_data_access_.get_custom_data(component);
   if (custom_data == nullptr) {
@@ -533,7 +575,7 @@ WriteAttributeLookup NamedLegacyCustomDataProvider::try_get_for_write(
   }
   for (CustomDataLayer &layer : MutableSpan(custom_data->layers, custom_data->totlayer)) {
     if (layer.type == stored_type_) {
-      if (layer.name == attribute_name) {
+      if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
         const int domain_size = component.attribute_domain_size(domain_);
         void *data_old = layer.data;
         void *data_new = CustomData_duplicate_referenced_layer_named(
@@ -549,7 +591,7 @@ WriteAttributeLookup NamedLegacyCustomDataProvider::try_get_for_write(
 }
 
 bool NamedLegacyCustomDataProvider::try_delete(GeometryComponent &component,
-                                               const StringRef attribute_name) const
+                                               const AttributeIDRef &attribute_id) const
 {
   CustomData *custom_data = custom_data_access_.get_custom_data(component);
   if (custom_data == nullptr) {
@@ -558,7 +600,7 @@ bool NamedLegacyCustomDataProvider::try_delete(GeometryComponent &component,
   for (const int i : IndexRange(custom_data->totlayer)) {
     const CustomDataLayer &layer = custom_data->layers[i];
     if (layer.type == stored_type_) {
-      if (layer.name == attribute_name) {
+      if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
         const int domain_size = component.attribute_domain_size(domain_);
         CustomData_free_layer(custom_data, stored_type_, domain_size, i);
         custom_data_access_.update_custom_data_pointers(component);
@@ -627,11 +669,11 @@ CustomDataAttributes &CustomDataAttributes::operator=(const CustomDataAttributes
   return *this;
 }
 
-std::optional<GSpan> CustomDataAttributes::get_for_read(const StringRef name) const
+std::optional<GSpan> CustomDataAttributes::get_for_read(const AttributeIDRef &attribute_id) const
 {
   BLI_assert(size_ != 0);
   for (const CustomDataLayer &layer : Span(data.layers, data.totlayer)) {
-    if (layer.name == name) {
+    if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       const CPPType *cpp_type = custom_data_type_to_cpp_type((CustomDataType)layer.type);
       BLI_assert(cpp_type != nullptr);
       return GSpan(*cpp_type, layer.data, size_);
@@ -645,13 +687,13 @@ std::optional<GSpan> CustomDataAttributes::get_for_read(const StringRef name) co
  * value if the attribute doesn't exist. If no default value is provided, the default value for the
  * type will be used.
  */
-GVArrayPtr CustomDataAttributes::get_for_read(const StringRef name,
+GVArrayPtr CustomDataAttributes::get_for_read(const AttributeIDRef &attribute_id,
                                               const CustomDataType data_type,
                                               const void *default_value) const
 {
   const CPPType *type = blender::bke::custom_data_type_to_cpp_type(data_type);
 
-  std::optional<GSpan> attribute = this->get_for_read(name);
+  std::optional<GSpan> attribute = this->get_for_read(attribute_id);
   if (!attribute) {
     const int domain_size = this->size_;
     return std::make_unique<GVArray_For_SingleValue>(
@@ -666,12 +708,12 @@ GVArrayPtr CustomDataAttributes::get_for_read(const StringRef name,
   return conversions.try_convert(std::make_unique<GVArray_For_GSpan>(*attribute), *type);
 }
 
-std::optional<GMutableSpan> CustomDataAttributes::get_for_write(const StringRef name)
+std::optional<GMutableSpan> CustomDataAttributes::get_for_write(const AttributeIDRef &attribute_id)
 {
   /* If this assert hits, it most likely means that #reallocate was not called at some point. */
   BLI_assert(size_ != 0);
   for (CustomDataLayer &layer : MutableSpan(data.layers, data.totlayer)) {
-    if (layer.name == name) {
+    if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       const CPPType *cpp_type = custom_data_type_to_cpp_type((CustomDataType)layer.type);
       BLI_assert(cpp_type != nullptr);
       return GMutableSpan(*cpp_type, layer.data, size_);
@@ -680,30 +722,29 @@ std::optional<GMutableSpan> CustomDataAttributes::get_for_write(const StringRef
   return {};
 }
 
-bool CustomDataAttributes::create(const StringRef name, const CustomDataType data_type)
+bool CustomDataAttributes::create(const AttributeIDRef &attribute_id,
+                                  const CustomDataType data_type)
 {
-  char name_c[MAX_NAME];
-  name.copy(name_c);
-  void *result = CustomData_add_layer_named(&data, data_type, CD_DEFAULT, nullptr, size_, name_c);
+  void *result = add_generic_custom_data_layer(
+      data, data_type, CD_DEFAULT, nullptr, size_, attribute_id);
   return result != nullptr;
 }
 
-bool CustomDataAttributes::create_by_move(const blender::StringRef name,
+bool CustomDataAttributes::create_by_move(const blender::bke::AttributeIDRef &attribute_id,
                                           const CustomDataType data_type,
                                           void *buffer)
 {
-  char name_c[MAX_NAME];
-  name.copy(name_c);
-  void *result = CustomData_add_layer_named(&data, data_type, CD_ASSIGN, buffer, size_, name_c);
+  void *result = add_generic_custom_data_layer(
+      data, data_type, CD_ASSIGN, buffer, size_, attribute_id);
   return result != nullptr;
 }
 
-bool CustomDataAttributes::remove(const blender::StringRef name)
+bool CustomDataAttributes::remove(const blender::bke::AttributeIDRef &attribute_id)
 {
   bool result = false;
   for (const int i : IndexRange(data.totlayer)) {
     const CustomDataLayer &layer = data.layers[i];
-    if (layer.name == name) {
+    if (custom_data_layer_matches_attribute_id(layer, attribute_id)) {
       CustomData_free_layer(&data, layer.type, size_, i);
       result = true;
     }
@@ -722,7 +763,14 @@ bool CustomDataAttributes::foreach_attribute(const AttributeForeachCallback call
 {
   for (const CustomDataLayer &layer : Span(data.layers, data.totlayer)) {
     AttributeMetaData meta_data{domain, (CustomDataType)layer.type};
-    if (!callback(layer.name, meta_data)) {
+    AttributeIDRef attribute_id;
+    if (layer.anonymous_id != nullptr) {
+      attribute_id = layer.anonymous_id;
+    }
+    else {
+      attribute_id = layer.name;
+    }
+    if (!callback(attribute_id, meta_data)) {
       return false;
     }
   }
@@ -766,21 +814,23 @@ bool GeometryComponent::attribute_is_builtin(const blender::StringRef attribute_
 }
 
 blender::bke::ReadAttributeLookup GeometryComponent::attribute_try_get_for_read(
-    const StringRef attribute_name) const
+    const blender::bke::AttributeIDRef &attribute_id) const
 {
   using namespace blender::bke;
   const ComponentAttributeProviders *providers = this->get_attribute_providers();
   if (providers == nullptr) {
     return {};
   }
-  const BuiltinAttributeProvider *builtin_provider =
-      providers->builtin_attribute_providers().lookup_default_as(attribute_name, nullptr);
-  if (builtin_provider != nullptr) {
-    return {builtin_provider->try_get_for_read(*this), builtin_provider->domain()};
+  if (attribute_id.is_named()) {
+    const BuiltinAttributeProvider *builtin_provider =
+        providers->builtin_attribute_providers().lookup_default_as(attribute_id.name(), nullptr);
+    if (builtin_provider != nullptr) {
+      return {builtin_provider->try_get_for_read(*this), builtin_provider->domain()};
+    }
   }
   for (const DynamicAttributesProvider *dynamic_provider :
        providers->dynamic_attribute_providers()) {
-    ReadAttributeLookup attribute = dynamic_provider->try_get_for_read(*this, attribute_name);
+    ReadAttributeLookup attribute = dynamic_provider->try_get_for_read(*this, attribute_id);
     if (attribute) {
       return attribute;
     }
@@ -800,21 +850,23 @@ std::unique_ptr<blender::fn::GVArray> GeometryComponent::attribute_try_adapt_dom
 }
 
 blender::bke::WriteAttributeLookup GeometryComponent::attribute_try_get_for_write(
-    const StringRef attribute_name)
+    const blender::bke::AttributeIDRef &attribute_id)
 {
   using namespace blender::bke;
   const ComponentAttributeProviders *providers = this->get_attribute_providers();
   if (providers == nullptr) {
     return {};
   }
-  const BuiltinAttributeProvider *builtin_provider =
-      providers->builtin_attribute_providers().lookup_default_as(attribute_name, nullptr);
-  if (builtin_provider != nullptr) {
-    return {builtin_provider->try_get_for_write(*this), builtin_provider->domain()};
+  if (attribute_id.is_named()) {
+    const BuiltinAttributeProvider *builtin_provider =
+        providers->builtin_attribute_providers().lookup_default_as(attribute_id.name(), nullptr);
+    if (builtin_provider != nullptr) {
+      return {builtin_provider->try_get_for_write(*this), builtin_provider->domain()};
+    }
   }
   for (const DynamicAttributesProvider *dynamic_provider :
        providers->dynamic_attribute_providers()) {
-    WriteAttributeLookup attribute = dynamic_provider->try_get_for_write(*this, attribute_name);
+    WriteAttributeLookup attribute = dynamic_provider->try_get_for_write(*this, attribute_id);
     if (attribute) {
       return attribute;
     }
@@ -822,53 +874,57 @@ blender::bke::WriteAttributeLookup GeometryComponent::attribute_try_get_for_writ
   return {};
 }
 
-bool GeometryComponent::attribute_try_delete(const StringRef attribute_name)
+bool GeometryComponent::attribute_try_delete(const blender::bke::AttributeIDRef &attribute_id)
 {
   using namespace blender::bke;
   const ComponentAttributeProviders *providers = this->get_attribute_providers();
   if (providers == nullptr) {
     return {};
   }
-  const BuiltinAttributeProvider *builtin_provider =
-      providers->builtin_attribute_providers().lookup_default_as(attribute_name, nullptr);
-  if (builtin_provider != nullptr) {
-    return builtin_provider->try_delete(*this);
+  if (attribute_id.is_named()) {
+    const BuiltinAttributeProvider *builtin_provider =
+        providers->builtin_attribute_providers().lookup_default_as(attribute_id.name(), nullptr);
+    if (builtin_provider != nullptr) {
+      return builtin_provider->try_delete(*this);
+    }
   }
   bool success = false;
   for (const DynamicAttributesProvider *dynamic_provider :
        providers->dynamic_attribute_providers()) {
-    success = dynamic_provider->try_delete(*this, attribute_name) || success;
+    success = dynamic_provider->try_delete(*this, attribute_id) || success;
   }
   return success;
 }
 
-bool GeometryComponent::attribute_try_create(const StringRef attribute_name,
+bool GeometryComponent::attribute_try_create(const blender::bke::AttributeIDRef &attribute_id,
                                              const AttributeDomain domain,
                                              const CustomDataType data_type,
                                              const AttributeInit &initializer)
 {
   using namespace blender::bke;
-  if (attribute_name.is_empty()) {
+  if (!attribute_id) {
     return false;
   }
   const ComponentAttributeProviders *providers = this->get_attribute_providers();
   if (providers == nullptr) {
     return false;
   }
-  const BuiltinAttributeProvider *builtin_provider =
-      providers->builtin_attribute_providers().lookup_default_as(attribute_name, nullptr);
-  if (builtin_provider != nullptr) {
-    if (builtin_provider->domain() != domain) {
-      return false;
+  if (attribute_id.is_named()) {
+    const BuiltinAttributeProvider *builtin_provider =
+        providers->builtin_attribute_providers().lookup_default_as(attribute_id.name(), nullptr);
+    if (builtin_provider != nullptr) {
+      if (builtin_provider->domain() != domain) {
+        return false;
+      }
+      if (builtin_provider->data_type() != data_type) {
+        return false;
+      }
+      return builtin_provider->try_create(*this, initializer);
     }
-    if (builtin_provider->data_type() != data_type) {
-      return false;
-    }
-    return builtin_provider->try_create(*this, initializer);
   }
   for (const DynamicAttributesProvider *dynamic_provider :
        providers->dynamic_attribute_providers()) {
-    if (dynamic_provider->try_create(*this, attribute_name, domain, data_type, initializer)) {
+    if (dynamic_provider->try_create(*this, attribute_id, domain, data_type, initializer)) {
       return true;
     }
   }
@@ -894,11 +950,12 @@ bool GeometryComponent::attribute_try_create_builtin(const blender::StringRef at
   return builtin_provider->try_create(*this, initializer);
 }
 
-Set<std::string> GeometryComponent::attribute_names() const
+Set<blender::bke::AttributeIDRef> GeometryComponent::attribute_ids() const
 {
-  Set<std::string> attributes;
-  this->attribute_foreach([&](StringRefNull name, const AttributeMetaData &UNUSED(meta_data)) {
-    attributes.add(name);
+  Set<blender::bke::AttributeIDRef> attributes;
+  this->attribute_foreach([&](const blender::bke::AttributeIDRef &attribute_id,
+                              const AttributeMetaData &UNUSED(meta_data)) {
+    attributes.add(attribute_id);
     return true;
   });
   return attributes;
@@ -931,9 +988,9 @@ bool GeometryComponent::attribute_foreach(const AttributeForeachCallback callbac
   }
   for (const DynamicAttributesProvider *provider : providers->dynamic_attribute_providers()) {
     const bool continue_loop = provider->foreach_attribute(
-        *this, [&](StringRefNull name, const AttributeMetaData &meta_data) {
-          if (handled_attribute_names.add(name)) {
-            return callback(name, meta_data);
+        *this, [&](const AttributeIDRef &attribute_id, const AttributeMetaData &meta_data) {
+          if (attribute_id.is_anonymous() || handled_attribute_names.add(attribute_id.name())) {
+            return callback(attribute_id, meta_data);
           }
           return true;
         });
@@ -945,9 +1002,9 @@ bool GeometryComponent::attribute_foreach(const AttributeForeachCallback callbac
   return true;
 }
 
-bool GeometryComponent::attribute_exists(const blender::StringRef attribute_name) const
+bool GeometryComponent::attribute_exists(const blender::bke::AttributeIDRef &attribute_id) const
 {
-  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_name);
+  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_id);
   if (attribute) {
     return true;
   }
@@ -955,11 +1012,12 @@ bool GeometryComponent::attribute_exists(const blender::StringRef attribute_name
 }
 
 std::optional<AttributeMetaData> GeometryComponent::attribute_get_meta_data(
-    const StringRef attribute_name) const
+    const blender::bke::AttributeIDRef &attribute_id) const
 {
   std::optional<AttributeMetaData> result{std::nullopt};
-  this->attribute_foreach([&](StringRefNull name, const AttributeMetaData &meta_data) {
-    if (attribute_name == name) {
+  this->attribute_foreach([&](const blender::bke::AttributeIDRef &current_attribute_id,
+                              const AttributeMetaData &meta_data) {
+    if (attribute_id == current_attribute_id) {
       result = meta_data;
       return false;
     }
@@ -977,11 +1035,11 @@ static std::unique_ptr<blender::fn::GVArray> try_adapt_data_type(
 }
 
 std::unique_ptr<blender::fn::GVArray> GeometryComponent::attribute_try_get_for_read(
-    const StringRef attribute_name,
+    const blender::bke::AttributeIDRef &attribute_id,
     const AttributeDomain domain,
     const CustomDataType data_type) const
 {
-  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_name);
+  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_id);
   if (!attribute) {
     return {};
   }
@@ -1007,13 +1065,13 @@ std::unique_ptr<blender::fn::GVArray> GeometryComponent::attribute_try_get_for_r
 }
 
 std::unique_ptr<blender::bke::GVArray> GeometryComponent::attribute_try_get_for_read(
-    const StringRef attribute_name, const AttributeDomain domain) const
+    const blender::bke::AttributeIDRef &attribute_id, const AttributeDomain domain) const
 {
   if (!this->attribute_domain_supported(domain)) {
     return {};
   }
 
-  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_name);
+  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_id);
   if (!attribute) {
     return {};
   }
@@ -1026,9 +1084,9 @@ std::unique_ptr<blender::bke::GVArray> GeometryComponent::attribute_try_get_for_
 }
 
 blender::bke::ReadAttributeLookup GeometryComponent::attribute_try_get_for_read(
-    const blender::StringRef attribute_name, const CustomDataType data_type) const
+    const blender::bke::AttributeIDRef &attribute_id, const CustomDataType data_type) const
 {
-  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_name);
+  blender::bke::ReadAttributeLookup attribute = this->attribute_try_get_for_read(attribute_id);
   if (!attribute) {
     return {};
   }
@@ -1043,13 +1101,13 @@ blender::bke::ReadAttributeLookup GeometryComponent::attribute_try_get_for_read(
 }
 
 std::unique_ptr<blender::bke::GVArray> GeometryComponent::attribute_get_for_read(
-    const StringRef attribute_name,
+    const blender::bke::AttributeIDRef &attribute_id,
     const AttributeDomain domain,
     const CustomDataType data_type,
     const void *default_value) const
 {
   std::unique_ptr<blender::bke::GVArray> varray = this->attribute_try_get_for_read(
-      attribute_name, domain, data_type);
+      attribute_id, domain, data_type);
   if (varray) {
     return varray;
   }
@@ -1065,15 +1123,22 @@ class GVMutableAttribute_For_OutputAttribute
     : public blender::fn::GVMutableArray_For_GMutableSpan {
  public:
   GeometryComponent *component;
-  std::string final_name;
+  std::string attribute_name;
+  blender::bke::WeakAnonymousAttributeID anonymous_attribute_id;
 
   GVMutableAttribute_For_OutputAttribute(GMutableSpan data,
                                          GeometryComponent &component,
-                                         std::string final_name)
-      : blender::fn::GVMutableArray_For_GMutableSpan(data),
-        component(&component),
-        final_name(std::move(final_name))
+                                         const blender::bke::AttributeIDRef &attribute_id)
+      : blender::fn::GVMutableArray_For_GMutableSpan(data), component(&component)
   {
+    if (attribute_id.is_named()) {
+      this->attribute_name = attribute_id.name();
+    }
+    else {
+      const AnonymousAttributeID *anonymous_id = &attribute_id.anonymous_id();
+      BKE_anonymous_attribute_id_increment_weak(anonymous_id);
+      this->anonymous_attribute_id = blender::bke::WeakAnonymousAttributeID{anonymous_id};
+    }
   }
 
   ~GVMutableAttribute_For_OutputAttribute() override
@@ -1093,21 +1158,28 @@ static void save_output_attribute(blender::bke::OutputAttribute &output_attribut
       dynamic_cast<GVMutableAttribute_For_OutputAttribute &>(output_attribute.varray());
 
   GeometryComponent &component = *varray.component;
-  const StringRefNull name = varray.final_name;
+  AttributeIDRef attribute_id;
+  if (!varray.attribute_name.empty()) {
+    attribute_id = varray.attribute_name;
+  }
+  else {
+    attribute_id = varray.anonymous_attribute_id.extract();
+  }
   const AttributeDomain domain = output_attribute.domain();
   const CustomDataType data_type = output_attribute.custom_data_type();
   const CPPType &cpp_type = output_attribute.cpp_type();
 
-  component.attribute_try_delete(name);
-  if (!component.attribute_try_create(
-          varray.final_name, domain, data_type, AttributeInitDefault())) {
-    CLOG_WARN(&LOG,
-              "Could not create the '%s' attribute with type '%s'.",
-              name.c_str(),
-              cpp_type.name().c_str());
+  component.attribute_try_delete(attribute_id);
+  if (!component.attribute_try_create(attribute_id, domain, data_type, AttributeInitDefault())) {
+    if (!varray.attribute_name.empty()) {
+      CLOG_WARN(&LOG,
+                "Could not create the '%s' attribute with type '%s'.",
+                varray.attribute_name.c_str(),
+                cpp_type.name().c_str());
+    }
     return;
   }
-  WriteAttributeLookup write_attribute = component.attribute_try_get_for_write(name);
+  WriteAttributeLookup write_attribute = component.attribute_try_get_for_write(attribute_id);
   BUFFER_FOR_CPP_TYPE_VALUE(varray.type(), buffer);
   for (const int i : IndexRange(varray.size())) {
     varray.get(i, buffer);
@@ -1117,7 +1189,7 @@ static void save_output_attribute(blender::bke::OutputAttribute &output_attribut
 
 static blender::bke::OutputAttribute create_output_attribute(
     GeometryComponent &component,
-    const blender::StringRef attribute_name,
+    const blender::bke::AttributeIDRef &attribute_id,
     const AttributeDomain domain,
     const CustomDataType data_type,
     const bool ignore_old_values,
@@ -1127,7 +1199,7 @@ static blender::bke::OutputAttribute create_output_attribute(
   using namespace blender::fn;
   using namespace blender::bke;
 
-  if (attribute_name.is_empty()) {
+  if (!attribute_id) {
     return {};
   }
 
@@ -1135,7 +1207,8 @@ static blender::bke::OutputAttribute create_output_attribute(
   BLI_assert(cpp_type != nullptr);
   const nodes::DataTypeConversions &conversions = nodes::get_implicit_type_conversions();
 
-  if (component.attribute_is_builtin(attribute_name)) {
+  if (attribute_id.is_named() && component.attribute_is_builtin(attribute_id.name())) {
+    const StringRef attribute_name = attribute_id.name();
     WriteAttributeLookup attribute = component.attribute_try_get_for_write(attribute_name);
     if (!attribute) {
       if (default_value) {
@@ -1169,18 +1242,18 @@ static blender::bke::OutputAttribute create_output_attribute(
 
   const int domain_size = component.attribute_domain_size(domain);
 
-  WriteAttributeLookup attribute = component.attribute_try_get_for_write(attribute_name);
+  WriteAttributeLookup attribute = component.attribute_try_get_for_write(attribute_id);
   if (!attribute) {
     if (default_value) {
       const GVArray_For_SingleValueRef default_varray{*cpp_type, domain_size, default_value};
       component.attribute_try_create(
-          attribute_name, domain, data_type, AttributeInitVArray(&default_varray));
+          attribute_id, domain, data_type, AttributeInitVArray(&default_varray));
     }
     else {
-      component.attribute_try_create(attribute_name, domain, data_type, AttributeInitDefault());
+      component.attribute_try_create(attribute_id, domain, data_type, AttributeInitDefault());
     }
 
-    attribute = component.attribute_try_get_for_write(attribute_name);
+    attribute = component.attribute_try_get_for_write(attribute_id);
     if (!attribute) {
       /* Can't create the attribute. */
       return {};
@@ -1202,28 +1275,28 @@ static blender::bke::OutputAttribute create_output_attribute(
   else {
     /* Fill the temporary array with values from the existing attribute. */
     GVArrayPtr old_varray = component.attribute_get_for_read(
-        attribute_name, domain, data_type, default_value);
+        attribute_id, domain, data_type, default_value);
     old_varray->materialize_to_uninitialized(IndexRange(domain_size), data);
   }
   GVMutableArrayPtr varray = std::make_unique<GVMutableAttribute_For_OutputAttribute>(
-      GMutableSpan{*cpp_type, data, domain_size}, component, attribute_name);
+      GMutableSpan{*cpp_type, data, domain_size}, component, attribute_id);
 
   return OutputAttribute(std::move(varray), domain, save_output_attribute, true);
 }
 
 blender::bke::OutputAttribute GeometryComponent::attribute_try_get_for_output(
-    const StringRef attribute_name,
+    const blender::bke::AttributeIDRef &attribute_id,
     const AttributeDomain domain,
     const CustomDataType data_type,
     const void *default_value)
 {
-  return create_output_attribute(*this, attribute_name, domain, data_type, false, default_value);
+  return create_output_attribute(*this, attribute_id, domain, data_type, false, default_value);
 }
 
 blender::bke::OutputAttribute GeometryComponent::attribute_try_get_for_output_only(
-    const blender::StringRef attribute_name,
+    const blender::bke::AttributeIDRef &attribute_id,
     const AttributeDomain domain,
     const CustomDataType data_type)
 {
-  return create_output_attribute(*this, attribute_name, domain, data_type, true, nullptr);
+  return create_output_attribute(*this, attribute_id, domain, data_type, true, nullptr);
 }

source/blender/blenkernel/intern/attribute_access_intern.hh

@@ -116,12 +116,13 @@ class BuiltinAttributeProvider {
 class DynamicAttributesProvider {
  public:
   virtual ReadAttributeLookup try_get_for_read(const GeometryComponent &component,
-                                               const StringRef attribute_name) const = 0;
+                                               const AttributeIDRef &attribute_id) const = 0;
   virtual WriteAttributeLookup try_get_for_write(GeometryComponent &component,
-                                                 const StringRef attribute_name) const = 0;
-  virtual bool try_delete(GeometryComponent &component, const StringRef attribute_name) const = 0;
+                                                 const AttributeIDRef &attribute_id) const = 0;
+  virtual bool try_delete(GeometryComponent &component,
+                          const AttributeIDRef &attribute_id) const = 0;
   virtual bool try_create(GeometryComponent &UNUSED(component),
-                          const StringRef UNUSED(attribute_name),
+                          const AttributeIDRef &UNUSED(attribute_id),
                           const AttributeDomain UNUSED(domain),
                           const CustomDataType UNUSED(data_type),
                           const AttributeInit &UNUSED(initializer)) const
@@ -154,15 +155,15 @@ class CustomDataAttributeProvider final : public DynamicAttributesProvider {
   }
 
   ReadAttributeLookup try_get_for_read(const GeometryComponent &component,
-                                       const StringRef attribute_name) const final;
+                                       const AttributeIDRef &attribute_id) const final;
 
   WriteAttributeLookup try_get_for_write(GeometryComponent &component,
-                                         const StringRef attribute_name) const final;
+                                         const AttributeIDRef &attribute_id) const final;
 
-  bool try_delete(GeometryComponent &component, const StringRef attribute_name) const final;
+  bool try_delete(GeometryComponent &component, const AttributeIDRef &attribute_id) const final;
 
   bool try_create(GeometryComponent &component,
-                  const StringRef attribute_name,
+                  const AttributeIDRef &attribute_id,
                   const AttributeDomain domain,
                   const CustomDataType data_type,
                   const AttributeInit &initializer) const final;
@@ -231,10 +232,10 @@ class NamedLegacyCustomDataProvider final : public DynamicAttributesProvider {
   }
 
   ReadAttributeLookup try_get_for_read(const GeometryComponent &component,
-                                       const StringRef attribute_name) const final;
+                                       const AttributeIDRef &attribute_id) const final;
   WriteAttributeLookup try_get_for_write(GeometryComponent &component,
-                                         const StringRef attribute_name) const final;
-  bool try_delete(GeometryComponent &component, const StringRef attribute_name) const final;
+                                         const AttributeIDRef &attribute_id) const final;
+  bool try_delete(GeometryComponent &component, const AttributeIDRef &attribute_id) const final;
   bool foreach_attribute(const GeometryComponent &component,
                          const AttributeForeachCallback callback) const final;
   void foreach_domain(const FunctionRef<void(AttributeDomain)> callback) const final;

source/blender/blenkernel/intern/bvhutils.cc

@@ -1254,11 +1254,10 @@ BVHTree *bvhtree_from_editmesh_looptri_ex(BVHTreeFromEditMesh *data,
     bool in_cache = bvhcache_find(
         bvh_cache_p, bvh_cache_type, &tree, &lock_started, mesh_eval_mutex);
     BVHCache *bvh_cache = *bvh_cache_p;
-    bvhtree_balance(tree, true);
-
     if (in_cache == false) {
       tree = bvhtree_from_editmesh_looptri_create_tree(
           epsilon, tree_type, axis, em, looptri_mask, looptri_num_active);
+      bvhtree_balance(tree, true);
 
       /* Save on cache for later use */
       // printf("BVHTree built and saved on cache\n");

source/blender/blenkernel/intern/cachefile.c

@@ -368,7 +368,7 @@ void BKE_cachefile_eval(Main *bmain, Depsgraph *depsgraph, CacheFile *cache_file
 #endif
 
   if (DEG_is_active(depsgraph)) {
-    /* Flush object paths back to original datablock for UI. */
+    /* Flush object paths back to original data-block for UI. */
     CacheFile *cache_file_orig = (CacheFile *)DEG_get_original_id(&cache_file->id);
     BLI_freelistN(&cache_file_orig->object_paths);
     BLI_duplicatelist(&cache_file_orig->object_paths, &cache_file->object_paths);
@@ -411,6 +411,12 @@ float BKE_cachefile_time_offset(const CacheFile *cache_file, const float time, c
   return cache_file->is_sequence ? frame : frame / fps - time_offset;
 }
 
+/**
+ * Determine whether the #CacheFile should use a render engine procedural. If so, data is not read
+ * from the file and bounding boxes are used to represent the objects in the Scene.
+ * Render engines will receive the bounding box as a placeholder but can instead
+ * load the data directly if they support it.
+ */
 bool BKE_cache_file_uses_render_procedural(const CacheFile *cache_file,
                                            Scene *scene,
                                            const int dag_eval_mode)

source/blender/blenkernel/intern/curve_eval.cc

@@ -25,6 +25,7 @@
 
 #include "DNA_curve_types.h"
 
+#include "BKE_anonymous_attribute.hh"
 #include "BKE_curve.h"
 #include "BKE_spline.hh"
 
@@ -330,13 +331,13 @@ void CurveEval::assert_valid_point_attributes() const
     return;
   }
   const int layer_len = splines_.first()->attributes.data.totlayer;
-  Map<StringRefNull, AttributeMetaData> map;
+  Map<blender::bke::AttributeIDRef, AttributeMetaData> map;
   for (const SplinePtr &spline : splines_) {
     BLI_assert(spline->attributes.data.totlayer == layer_len);
     spline->attributes.foreach_attribute(
-        [&](StringRefNull name, const AttributeMetaData &meta_data) {
+        [&](const blender::bke::AttributeIDRef &attribute_id, const AttributeMetaData &meta_data) {
           map.add_or_modify(
-              name,
+              attribute_id,
               [&](AttributeMetaData *map_data) {
                 /* All unique attribute names should be added on the first spline. */
                 BLI_assert(spline == splines_.first());

source/blender/blenkernel/intern/customdata.c

@@ -46,6 +46,7 @@
 
 #include "BLT_translation.h"
 
+#include "BKE_anonymous_attribute.h"
 #include "BKE_customdata.h"
 #include "BKE_customdata_file.h"
 #include "BKE_deform.h"
@@ -2127,6 +2128,10 @@ bool CustomData_merge(const struct CustomData *source,
     if (flag & CD_FLAG_NOCOPY) {
       continue;
     }
+    if (layer->anonymous_id &&
+        !BKE_anonymous_attribute_id_has_strong_references(layer->anonymous_id)) {
+      continue;
+    }
     if (!(mask & CD_TYPE_AS_MASK(type))) {
       continue;
     }
@@ -2166,6 +2171,11 @@ bool CustomData_merge(const struct CustomData *source,
       newlayer->active_mask = lastmask;
       newlayer->flag |= flag & (CD_FLAG_EXTERNAL | CD_FLAG_IN_MEMORY);
       changed = true;
+
+      if (layer->anonymous_id != NULL) {
+        BKE_anonymous_attribute_id_increment_weak(layer->anonymous_id);
+        newlayer->anonymous_id = layer->anonymous_id;
+      }
     }
   }
 
@@ -2206,6 +2216,10 @@ static void customData_free_layer__internal(CustomDataLayer *layer, int totelem)
 {
   const LayerTypeInfo *typeInfo;
 
+  if (layer->anonymous_id != NULL) {
+    BKE_anonymous_attribute_id_decrement_weak(layer->anonymous_id);
+    layer->anonymous_id = NULL;
+  }
   if (!(layer->flag & CD_FLAG_NOFREE) && layer->data) {
     typeInfo = layerType_getInfo(layer->type);
 
@@ -2649,6 +2663,27 @@ void *CustomData_add_layer_named(CustomData *data,
   return NULL;
 }
 
+void *CustomData_add_layer_anonymous(struct CustomData *data,
+                                     int type,
+                                     eCDAllocType alloctype,
+                                     void *layerdata,
+                                     int totelem,
+                                     const AnonymousAttributeID *anonymous_id)
+{
+  const char *name = BKE_anonymous_attribute_id_internal_name(anonymous_id);
+  CustomDataLayer *layer = customData_add_layer__internal(
+      data, type, alloctype, layerdata, totelem, name);
+  CustomData_update_typemap(data);
+
+  if (layer == NULL) {
+    return NULL;
+  }
+
+  BKE_anonymous_attribute_id_increment_weak(anonymous_id);
+  layer->anonymous_id = anonymous_id;
+  return layer->data;
+}
+
 bool CustomData_free_layer(CustomData *data, int type, int totelem, int index)
 {
   const int index_first = CustomData_get_layer_index(data, type);
@@ -2812,6 +2847,20 @@ void *CustomData_duplicate_referenced_layer_named(CustomData *data,
   return customData_duplicate_referenced_layer_index(data, layer_index, totelem);
 }
 
+void *CustomData_duplicate_referenced_layer_anonymous(CustomData *data,
+                                                      const int type,
+                                                      const AnonymousAttributeID *anonymous_id,
+                                                      const int totelem)
+{
+  for (int i = 0; i < data->totlayer; i++) {
+    if (data->layers[i].anonymous_id == anonymous_id) {
+      return customData_duplicate_referenced_layer_index(data, i, totelem);
+    }
+  }
+  BLI_assert_unreachable();
+  return NULL;
+}
+
 void CustomData_duplicate_referenced_layers(CustomData *data, int totelem)
 {
   for (int i = 0; i < data->totlayer; i++) {
@@ -4244,7 +4293,8 @@ void CustomData_blend_write_prepare(CustomData *data,
 
   for (i = 0, j = 0; i < totlayer; i++) {
     CustomDataLayer *layer = &data->layers[i];
-    if (layer->flag & CD_FLAG_NOCOPY) { /* Layers with this flag set are not written to file. */
+    /* Layers with this flag set are not written to file. */
+    if ((layer->flag & CD_FLAG_NOCOPY) || layer->anonymous_id != NULL) {
       data->totlayer--;
       // CLOG_WARN(&LOG, "skipping layer %p (%s)", layer, layer->name);
     }

source/blender/blenkernel/intern/geometry_component_curve.cc

@@ -892,7 +892,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
 
  public:
   ReadAttributeLookup try_get_for_read(const GeometryComponent &component,
-                                       const StringRef attribute_name) const final
+                                       const AttributeIDRef &attribute_id) const final
   {
     const CurveEval *curve = get_curve_from_component_for_read(component);
     if (curve == nullptr || curve->splines().size() == 0) {
@@ -902,13 +902,13 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
     Span<SplinePtr> splines = curve->splines();
     Vector<GSpan> spans; /* GSpan has no default constructor. */
     spans.reserve(splines.size());
-    std::optional<GSpan> first_span = splines[0]->attributes.get_for_read(attribute_name);
+    std::optional<GSpan> first_span = splines[0]->attributes.get_for_read(attribute_id);
     if (!first_span) {
       return {};
     }
     spans.append(*first_span);
     for (const int i : IndexRange(1, splines.size() - 1)) {
-      std::optional<GSpan> span = splines[i]->attributes.get_for_read(attribute_name);
+      std::optional<GSpan> span = splines[i]->attributes.get_for_read(attribute_id);
       if (!span) {
         /* All splines should have the same set of data layers. It would be possible to recover
          * here and return partial data instead, but that would add a lot of complexity for a
@@ -945,7 +945,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
 
   /* This function is almost the same as #try_get_for_read, but without const. */
   WriteAttributeLookup try_get_for_write(GeometryComponent &component,
-                                         const StringRef attribute_name) const final
+                                         const AttributeIDRef &attribute_id) const final
   {
     CurveEval *curve = get_curve_from_component_for_write(component);
     if (curve == nullptr || curve->splines().size() == 0) {
@@ -955,13 +955,13 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
     MutableSpan<SplinePtr> splines = curve->splines();
     Vector<GMutableSpan> spans; /* GMutableSpan has no default constructor. */
     spans.reserve(splines.size());
-    std::optional<GMutableSpan> first_span = splines[0]->attributes.get_for_write(attribute_name);
+    std::optional<GMutableSpan> first_span = splines[0]->attributes.get_for_write(attribute_id);
     if (!first_span) {
       return {};
     }
     spans.append(*first_span);
     for (const int i : IndexRange(1, splines.size() - 1)) {
-      std::optional<GMutableSpan> span = splines[i]->attributes.get_for_write(attribute_name);
+      std::optional<GMutableSpan> span = splines[i]->attributes.get_for_write(attribute_id);
       if (!span) {
         /* All splines should have the same set of data layers. It would be possible to recover
          * here and return partial data instead, but that would add a lot of complexity for a
@@ -996,7 +996,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
     return attribute;
   }
 
-  bool try_delete(GeometryComponent &component, const StringRef attribute_name) const final
+  bool try_delete(GeometryComponent &component, const AttributeIDRef &attribute_id) const final
   {
     CurveEval *curve = get_curve_from_component_for_write(component);
     if (curve == nullptr) {
@@ -1006,7 +1006,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
     /* Reuse the boolean for all splines; we expect all splines to have the same attributes. */
     bool layer_freed = false;
     for (SplinePtr &spline : curve->splines()) {
-      layer_freed = spline->attributes.remove(attribute_name);
+      layer_freed = spline->attributes.remove(attribute_id);
     }
     return layer_freed;
   }
@@ -1034,7 +1034,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
   }
 
   bool try_create(GeometryComponent &component,
-                  const StringRef attribute_name,
+                  const AttributeIDRef &attribute_id,
                   const AttributeDomain domain,
                   const CustomDataType data_type,
                   const AttributeInit &initializer) const final
@@ -1053,7 +1053,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
     /* First check the one case that allows us to avoid copying the input data. */
     if (splines.size() == 1 && initializer.type == AttributeInit::Type::MoveArray) {
       void *source_data = static_cast<const AttributeInitMove &>(initializer).data;
-      if (!splines[0]->attributes.create_by_move(attribute_name, data_type, source_data)) {
+      if (!splines[0]->attributes.create_by_move(attribute_id, data_type, source_data)) {
         MEM_freeN(source_data);
         return false;
       }
@@ -1062,7 +1062,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
 
     /* Otherwise just create a custom data layer on each of the splines. */
     for (const int i : splines.index_range()) {
-      if (!splines[i]->attributes.create(attribute_name, data_type)) {
+      if (!splines[i]->attributes.create(attribute_id, data_type)) {
         /* If attribute creation fails on one of the splines, we cannot leave the custom data
          * layers in the previous splines around, so delete them before returning. However,
          * this is not an expected case. */
@@ -1076,7 +1076,7 @@ class DynamicPointAttributeProvider final : public DynamicAttributesProvider {
       return true;
     }
 
-    WriteAttributeLookup write_attribute = this->try_get_for_write(component, attribute_name);
+    WriteAttributeLookup write_attribute = this->try_get_for_write(component, attribute_id);
     /* We just created the attribute, it should exist. */
     BLI_assert(write_attribute);
 

source/blender/blenkernel/intern/geometry_component_mesh.cc

@@ -818,16 +818,20 @@ class VArray_For_VertexWeights final : public VArray<float> {
 class VertexGroupsAttributeProvider final : public DynamicAttributesProvider {
  public:
   ReadAttributeLookup try_get_for_read(const GeometryComponent &component,
-                                       const StringRef attribute_name) const final
+                                       const AttributeIDRef &attribute_id) const final
   {
     BLI_assert(component.type() == GEO_COMPONENT_TYPE_MESH);
+    if (!attribute_id.is_named()) {
+      return {};
+    }
     const MeshComponent &mesh_component = static_cast<const MeshComponent &>(component);
     const Mesh *mesh = mesh_component.get_for_read();
     if (mesh == nullptr) {
       return {};
     }
+    std::string name = attribute_id.name();
     const int vertex_group_index = BLI_findstringindex(
-        &mesh->vertex_group_names, attribute_name.data(), offsetof(bDeformGroup, name));
+        &mesh->vertex_group_names, name.c_str(), offsetof(bDeformGroup, name));
     if (vertex_group_index < 0) {
       return {};
     }
@@ -843,17 +847,21 @@ class VertexGroupsAttributeProvider final : public DynamicAttributesProvider {
   }
 
   WriteAttributeLookup try_get_for_write(GeometryComponent &component,
-                                         const StringRef attribute_name) const final
+                                         const AttributeIDRef &attribute_id) const final
   {
     BLI_assert(component.type() == GEO_COMPONENT_TYPE_MESH);
+    if (!attribute_id.is_named()) {
+      return {};
+    }
     MeshComponent &mesh_component = static_cast<MeshComponent &>(component);
     Mesh *mesh = mesh_component.get_for_write();
     if (mesh == nullptr) {
       return {};
     }
 
+    const std::string name = attribute_id.name();
     const int vertex_group_index = BLI_findstringindex(
-        &mesh->vertex_group_names, attribute_name.data(), offsetof(bDeformGroup, name));
+        &mesh->vertex_group_names, name.c_str(), offsetof(bDeformGroup, name));
     if (vertex_group_index < 0) {
       return {};
     }
@@ -872,17 +880,21 @@ class VertexGroupsAttributeProvider final : public DynamicAttributesProvider {
         ATTR_DOMAIN_POINT};
   }
 
-  bool try_delete(GeometryComponent &component, const StringRef attribute_name) const final
+  bool try_delete(GeometryComponent &component, const AttributeIDRef &attribute_id) const final
   {
     BLI_assert(component.type() == GEO_COMPONENT_TYPE_MESH);
+    if (!attribute_id.is_named()) {
+      return false;
+    }
     MeshComponent &mesh_component = static_cast<MeshComponent &>(component);
     Mesh *mesh = mesh_component.get_for_write();
     if (mesh == nullptr) {
       return true;
     }
 
+    const std::string name = attribute_id.name();
     const int vertex_group_index = BLI_findstringindex(
-        &mesh->vertex_group_names, attribute_name.data(), offsetof(bDeformGroup, name));
+        &mesh->vertex_group_names, name.c_str(), offsetof(bDeformGroup, name));
     if (vertex_group_index < 0) {
       return false;
     }

source/blender/blenkernel/intern/geometry_set_instances.cc

@@ -319,7 +319,7 @@ void geometry_set_instances_attribute_foreach(const GeometrySet &geometry_set,
 void geometry_set_gather_instances_attribute_info(Span<GeometryInstanceGroup> set_groups,
                                                   Span<GeometryComponentType> component_types,
                                                   const Set<std::string> &ignored_attributes,
-                                                  Map<std::string, AttributeKind> &r_attributes)
+                                                  Map<AttributeIDRef, AttributeKind> &r_attributes)
 {
   for (const GeometryInstanceGroup &set_group : set_groups) {
     const GeometrySet &set = set_group.geometry_set;
@@ -329,23 +329,24 @@ void geometry_set_gather_instances_attribute_info(Span<GeometryInstanceGroup> se
       }
       const GeometryComponent &component = *set.get_component_for_read(component_type);
 
-      component.attribute_foreach([&](StringRefNull name, const AttributeMetaData &meta_data) {
-        if (ignored_attributes.contains(name)) {
-          return true;
-        }
-        auto add_info = [&](AttributeKind *attribute_kind) {
-          attribute_kind->domain = meta_data.domain;
-          attribute_kind->data_type = meta_data.data_type;
-        };
-        auto modify_info = [&](AttributeKind *attribute_kind) {
-          attribute_kind->domain = meta_data.domain; /* TODO: Use highest priority domain. */
-          attribute_kind->data_type = bke::attribute_data_type_highest_complexity(
-              {attribute_kind->data_type, meta_data.data_type});
-        };
+      component.attribute_foreach(
+          [&](const AttributeIDRef &attribute_id, const AttributeMetaData &meta_data) {
+            if (attribute_id.is_named() && ignored_attributes.contains(attribute_id.name())) {
+              return true;
+            }
+            auto add_info = [&](AttributeKind *attribute_kind) {
+              attribute_kind->domain = meta_data.domain;
+              attribute_kind->data_type = meta_data.data_type;
+            };
+            auto modify_info = [&](AttributeKind *attribute_kind) {
+              attribute_kind->domain = meta_data.domain; /* TODO: Use highest priority domain. */
+              attribute_kind->data_type = bke::attribute_data_type_highest_complexity(
+                  {attribute_kind->data_type, meta_data.data_type});
+            };
 
-        r_attributes.add_or_modify(name, add_info, modify_info);
-        return true;
-      });
+            r_attributes.add_or_modify(attribute_id, add_info, modify_info);
+            return true;
+          });
     }
   }
 }
@@ -500,11 +501,11 @@ static Mesh *join_mesh_topology_and_builtin_attributes(Span<GeometryInstanceGrou
 
 static void join_attributes(Span<GeometryInstanceGroup> set_groups,
                             Span<GeometryComponentType> component_types,
-                            const Map<std::string, AttributeKind> &attribute_info,
+                            const Map<AttributeIDRef, AttributeKind> &attribute_info,
                             GeometryComponent &result)
 {
-  for (Map<std::string, AttributeKind>::Item entry : attribute_info.items()) {
-    StringRef name = entry.key;
+  for (Map<AttributeIDRef, AttributeKind>::Item entry : attribute_info.items()) {
+    const AttributeIDRef attribute_id = entry.key;
     const AttributeDomain domain_output = entry.value.domain;
     const CustomDataType data_type_output = entry.value.data_type;
     const CPPType *cpp_type = bke::custom_data_type_to_cpp_type(data_type_output);
@@ -512,7 +513,7 @@ static void join_attributes(Span<GeometryInstanceGroup> set_groups,
 
     result.attribute_try_create(
         entry.key, domain_output, data_type_output, AttributeInitDefault());
-    WriteAttributeLookup write_attribute = result.attribute_try_get_for_write(name);
+    WriteAttributeLookup write_attribute = result.attribute_try_get_for_write(attribute_id);
     if (!write_attribute || &write_attribute.varray->type() != cpp_type ||
         write_attribute.domain != domain_output) {
       continue;
@@ -531,7 +532,7 @@ static void join_attributes(Span<GeometryInstanceGroup> set_groups,
             continue; /* Domain size is 0, so no need to increment the offset. */
           }
           GVArrayPtr source_attribute = component.attribute_try_get_for_read(
-              name, domain_output, data_type_output);
+              attribute_id, domain_output, data_type_output);
 
           if (source_attribute) {
             fn::GVArray_GSpan src_span{*source_attribute};
@@ -641,7 +642,7 @@ static void join_instance_groups_mesh(Span<GeometryInstanceGroup> set_groups,
   }
 
   /* Don't copy attributes that are stored directly in the mesh data structs. */
-  Map<std::string, AttributeKind> attributes;
+  Map<AttributeIDRef, AttributeKind> attributes;
   geometry_set_gather_instances_attribute_info(
       set_groups,
       component_types,
@@ -662,7 +663,7 @@ static void join_instance_groups_pointcloud(Span<GeometryInstanceGroup> set_grou
   PointCloudComponent &dst_component = result.get_component_for_write<PointCloudComponent>();
   dst_component.replace(new_pointcloud);
 
-  Map<std::string, AttributeKind> attributes;
+  Map<AttributeIDRef, AttributeKind> attributes;
   geometry_set_gather_instances_attribute_info(
       set_groups, {GEO_COMPONENT_TYPE_POINT_CLOUD}, {"position"}, attributes);
   join_attributes(set_groups,
@@ -696,7 +697,7 @@ static void join_instance_groups_curve(Span<GeometryInstanceGroup> set_groups, G
   CurveComponent &dst_component = result.get_component_for_write<CurveComponent>();
   dst_component.replace(curve);
 
-  Map<std::string, AttributeKind> attributes;
+  Map<AttributeIDRef, AttributeKind> attributes;
   geometry_set_gather_instances_attribute_info(
       set_groups,
       {GEO_COMPONENT_TYPE_CURVE},

source/blender/blenkernel/intern/gpencil_geom.cc

@@ -620,7 +620,10 @@ bool BKE_gpencil_stroke_trim_points(bGPDstroke *gps, const int index_from, const
   }
 
   if (new_count == 1) {
-    BKE_gpencil_free_stroke_weights(gps);
+    if (gps->dvert) {
+      BKE_gpencil_free_stroke_weights(gps);
+      MEM_freeN(gps->dvert);
+    }
     MEM_freeN(gps->points);
     gps->points = nullptr;
     gps->dvert = nullptr;
@@ -628,27 +631,24 @@ bool BKE_gpencil_stroke_trim_points(bGPDstroke *gps, const int index_from, const
     return false;
   }
 
-  new_pt = (bGPDspoint *)MEM_callocN(sizeof(bGPDspoint) * new_count, "gp_stroke_points_trimmed");
-
-  for (int i = 0; i < new_count; i++) {
-    memcpy(&new_pt[i], &pt[i + index_from], sizeof(bGPDspoint));
-  }
+  new_pt = (bGPDspoint *)MEM_mallocN(sizeof(bGPDspoint) * new_count, "gp_stroke_points_trimmed");
+  memcpy(new_pt, &pt[index_from], sizeof(bGPDspoint) * new_count);
 
   if (gps->dvert) {
-    new_dv = (MDeformVert *)MEM_callocN(sizeof(MDeformVert) * new_count,
+    new_dv = (MDeformVert *)MEM_mallocN(sizeof(MDeformVert) * new_count,
                                         "gp_stroke_dverts_trimmed");
     for (int i = 0; i < new_count; i++) {
       dv = &gps->dvert[i + index_from];
       new_dv[i].flag = dv->flag;
       new_dv[i].totweight = dv->totweight;
-      new_dv[i].dw = (MDeformWeight *)MEM_callocN(sizeof(MDeformWeight) * dv->totweight,
+      new_dv[i].dw = (MDeformWeight *)MEM_mallocN(sizeof(MDeformWeight) * dv->totweight,
                                                   "gp_stroke_dverts_dw_trimmed");
       for (int j = 0; j < dv->totweight; j++) {
         new_dv[i].dw[j].weight = dv->dw[j].weight;
         new_dv[i].dw[j].def_nr = dv->dw[j].def_nr;
       }
-      BKE_defvert_clear(dv);
     }
+    BKE_gpencil_free_stroke_weights(gps);
     MEM_freeN(gps->dvert);
     gps->dvert = new_dv;
   }
@@ -692,25 +692,21 @@ bool BKE_gpencil_stroke_split(bGPdata *gpd,
       gpf, gps, gps->mat_nr, new_count, gps->thickness);
 
   new_pt = new_gps->points; /* Allocated from above. */
-
-  for (int i = 0; i < new_count; i++) {
-    memcpy(&new_pt[i], &pt[i + before_index], sizeof(bGPDspoint));
-  }
+  memcpy(new_pt, &pt[before_index], sizeof(bGPDspoint) * new_count);
 
   if (gps->dvert) {
-    new_dv = (MDeformVert *)MEM_callocN(sizeof(MDeformVert) * new_count,
+    new_dv = (MDeformVert *)MEM_mallocN(sizeof(MDeformVert) * new_count,
                                         "gp_stroke_dverts_remaining(MDeformVert)");
     for (int i = 0; i < new_count; i++) {
       dv = &gps->dvert[i + before_index];
       new_dv[i].flag = dv->flag;
       new_dv[i].totweight = dv->totweight;
-      new_dv[i].dw = (MDeformWeight *)MEM_callocN(sizeof(MDeformWeight) * dv->totweight,
+      new_dv[i].dw = (MDeformWeight *)MEM_mallocN(sizeof(MDeformWeight) * dv->totweight,
                                                   "gp_stroke_dverts_dw_remaining(MDeformWeight)");
       for (int j = 0; j < dv->totweight; j++) {
         new_dv[i].dw[j].weight = dv->dw[j].weight;
         new_dv[i].dw[j].def_nr = dv->dw[j].def_nr;
       }
-      BKE_defvert_clear(dv);
     }
     new_gps->dvert = new_dv;
   }

source/blender/blenkernel/intern/lib_id.c

@@ -1481,7 +1481,7 @@ static bool id_name_final_build(char *name, char *base_name, size_t base_name_le
 
     /* Code above may have generated invalid utf-8 string, due to raw truncation.
      * Ensure we get a valid one now. */
-    base_name_len -= (size_t)BLI_utf8_invalid_strip(base_name, base_name_len);
+    base_name_len -= (size_t)BLI_str_utf8_invalid_strip(base_name, base_name_len);
 
     /* Also truncate orig name, and start the whole check again. */
     name[base_name_len] = '\0';
@@ -1731,7 +1731,7 @@ bool BKE_id_new_name_validate(ListBase *lb, ID *id, const char *tname, const boo
   else {
     /* disallow non utf8 chars,
      * the interface checks for this but new ID's based on file names don't */
-    BLI_utf8_invalid_strip(name, strlen(name));
+    BLI_str_utf8_invalid_strip(name, strlen(name));
   }
 
   ID *id_sorting_hint = NULL;

source/blender/blenkernel/intern/simulation.cc

@@ -49,14 +49,10 @@
 #include "BKE_simulation.h"
 
 #include "NOD_geometry.h"
-#include "NOD_node_tree_multi_function.hh"
 
 #include "BLI_map.hh"
 #include "BLT_translation.h"
 
-#include "FN_multi_function_network_evaluation.hh"
-#include "FN_multi_function_network_optimization.hh"
-
 #include "DEG_depsgraph.h"
 #include "DEG_depsgraph_query.h"
 

source/blender/blenkernel/intern/text.c

@@ -308,7 +308,7 @@ int txt_extended_ascii_as_utf8(char **str)
   int added = 0;
 
   while ((*str)[i]) {
-    if ((bad_char = BLI_utf8_invalid_byte(*str + i, length - i)) == -1) {
+    if ((bad_char = BLI_str_utf8_invalid_byte(*str + i, length - i)) == -1) {
       break;
     }
 
@@ -322,7 +322,7 @@ int txt_extended_ascii_as_utf8(char **str)
     i = 0;
 
     while ((*str)[i]) {
-      if ((bad_char = BLI_utf8_invalid_byte((*str) + i, length - i)) == -1) {
+      if ((bad_char = BLI_str_utf8_invalid_byte((*str) + i, length - i)) == -1) {
         memcpy(newstr + mi, (*str) + i, length - i + 1);
         break;
       }

source/blender/blenlib/BLI_fileops.h

@@ -154,18 +154,17 @@ bool BLI_file_is_writable(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL
 bool BLI_file_touch(const char *file) ATTR_NONNULL();
 bool BLI_file_alias_target(const char *filepath, char *r_targetpath) ATTR_WARN_UNUSED_RESULT;
 
-#if 0 /* UNUSED */
-int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
-#endif
-char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT
-    ATTR_NONNULL();
-size_t BLI_gzip_mem_to_file_at_pos(void *buf,
-                                   size_t len,
-                                   FILE *file,
-                                   size_t gz_stream_offset,
-                                   int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
-size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+bool BLI_file_magic_is_gzip(const char header[4]);
+
+size_t BLI_file_zstd_from_mem_at_pos(void *buf,
+                                     size_t len,
+                                     FILE *file,
+                                     size_t file_offset,
+                                     int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
     ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+bool BLI_file_magic_is_zstd(const char header[4]);
+
 size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
 size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 

source/blender/blenlib/BLI_filereader.h

@@ -0,0 +1,81 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ * \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
+ */
+
+#pragma once
+
+#ifdef WIN32
+#  include "BLI_winstuff.h"
+#else
+#  include <sys/types.h>
+#endif
+
+#include "BLI_compiler_attrs.h"
+#include "BLI_utildefines.h"
+
+#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
+typedef int64_t off64_t;
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FileReader;
+
+typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
+typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
+typedef void (*FileReaderCloseFn)(struct FileReader *reader);
+
+/* General structure for all FileReaders, implementations add custom fields at the end. */
+typedef struct FileReader {
+  FileReaderReadFn read;
+  FileReaderSeekFn seek;
+  FileReaderCloseFn close;
+
+  off64_t offset;
+} FileReader;
+
+/* Functions for opening the various types of FileReader.
+ * They either succeed and return a valid FileReader, or fail and return NULL.
+ *
+ * If a FileReader is created, it has to be cleaned up and freed by calling
+ * its close() function unless another FileReader has taken ownership - for example,
+ * Zstd and Gzip take over the base FileReader and will clean it up when their clean() is called.
+ */
+
+/* Create FileReader from raw file descriptor. */
+FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
+/* Create FileReader from raw file descriptor using memory-mapped IO. */
+FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
+/* Create FileReader from a region of memory. */
+FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
+    ATTR_NONNULL();
+/* Create FileReader from applying Zstd decompression on an underlying file. */
+FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+/* Create FileReader from applying Gzip decompression on an underlying file. */
+FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+
+#ifdef __cplusplus
+}
+#endif

source/blender/blenlib/BLI_math_base.h

@@ -196,6 +196,8 @@ MINLINE unsigned int log2_ceil_u(unsigned int x);
 MINLINE int divide_round_i(int a, int b);
 MINLINE int mod_i(int i, int n);
 
+MINLINE float round_to_even(float f);
+
 MINLINE signed char round_fl_to_char(float a);
 MINLINE unsigned char round_fl_to_uchar(float a);
 MINLINE short round_fl_to_short(float a);

source/blender/blenlib/BLI_string_utf8.h

@@ -31,8 +31,8 @@ char *BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t
     ATTR_NONNULL();
 size_t BLI_strncpy_utf8_rlen(char *__restrict dst, const char *__restrict src, size_t maxncpy)
     ATTR_NONNULL();
-ptrdiff_t BLI_utf8_invalid_byte(const char *str, size_t length) ATTR_NONNULL();
-int BLI_utf8_invalid_strip(char *str, size_t length) ATTR_NONNULL();
+ptrdiff_t BLI_str_utf8_invalid_byte(const char *str, size_t length) ATTR_NONNULL();
+int BLI_str_utf8_invalid_strip(char *str, size_t length) ATTR_NONNULL();
 
 /* warning, can return -1 on bad chars */
 int BLI_str_utf8_size(const char *p) ATTR_NONNULL();

source/blender/blenlib/CMakeLists.txt

@@ -31,6 +31,7 @@ set(INC
 
 set(INC_SYS
   ${ZLIB_INCLUDE_DIRS}
+  ${ZSTD_INCLUDE_DIRS}
   ${FREETYPE_INCLUDE_DIRS}
   ${GMP_INCLUDE_DIRS}
 )
@@ -75,6 +76,10 @@ set(SRC
   intern/endian_switch.c
   intern/expr_pylike_eval.c
   intern/fileops.c
+  intern/filereader_file.c
+  intern/filereader_gzip.c
+  intern/filereader_memory.c
+  intern/filereader_zstd.c
   intern/fnmatch.c
   intern/freetypefont.c
   intern/gsqueue.c
@@ -194,6 +199,7 @@ set(SRC
   BLI_enumerable_thread_specific.hh
   BLI_expr_pylike_eval.h
   BLI_fileops.h
+  BLI_filereader.h
   BLI_fileops_types.h
   BLI_float2.hh
   BLI_float3.hh
@@ -323,6 +329,7 @@ set(LIB
 
   ${FREETYPE_LIBRARY}
   ${ZLIB_LIBRARIES}
+  ${ZSTD_LIBRARIES}
 )
 
 if(WITH_MEM_VALGRIND)

source/blender/blenlib/intern/fileops.c

@@ -31,6 +31,7 @@
 #include <errno.h>
 
 #include "zlib.h"
+#include "zstd.h"
 
 #ifdef WIN32
 #  include "BLI_fileops_types.h"
@@ -61,199 +62,123 @@
 #include "BLI_sys_types.h" /* for intptr_t support */
 #include "BLI_utildefines.h"
 
-#if 0 /* UNUSED */
-/* gzip the file in from and write it to "to".
- * return -1 if zlib fails, -2 if the originating file does not exist
- * NOTE: will remove the "from" file
- */
-int BLI_file_gzip(const char *from, const char *to)
+size_t BLI_file_zstd_from_mem_at_pos(
+    void *buf, size_t len, FILE *file, size_t file_offset, int compression_level)
 {
-  char buffer[10240];
-  int file;
-  int readsize = 0;
-  int rval = 0, err;
-  gzFile gzfile;
+  fseek(file, file_offset, SEEK_SET);
 
-  /* level 1 is very close to 3 (the default) in terms of file size,
-   * but about twice as fast, best use for speedy saving - campbell */
-  gzfile = BLI_gzopen(to, "wb1");
-  if (gzfile == NULL) {
-    return -1;
-  }
-  file = BLI_open(from, O_BINARY | O_RDONLY, 0);
-  if (file == -1) {
-    return -2;
-  }
+  ZSTD_CCtx *ctx = ZSTD_createCCtx();
+  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
 
-  while (1) {
-    readsize = read(file, buffer, sizeof(buffer));
+  ZSTD_inBuffer input = {buf, len, 0};
 
-    if (readsize < 0) {
-      rval = -2; /* error happened in reading */
-      fprintf(stderr, "Error reading file %s: %s.\n", from, strerror(errno));
+  size_t out_len = ZSTD_CStreamOutSize();
+  void *out_buf = MEM_mallocN(out_len, __func__);
+  size_t total_written = 0;
+
+  /* Compress block and write it out until the input has been consumed. */
+  while (input.pos < input.size) {
+    ZSTD_outBuffer output = {out_buf, out_len, 0};
+    size_t ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_continue);
+    if (ZSTD_isError(ret)) {
       break;
     }
-    else if (readsize == 0) {
-      break; /* done reading */
-    }
-
-    if (gzwrite(gzfile, buffer, readsize) <= 0) {
-      rval = -1; /* error happened in writing */
-      fprintf(stderr, "Error writing gz file %s: %s.\n", to, gzerror(gzfile, &err));
+    if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
       break;
     }
+    total_written += output.pos;
   }
 
-  gzclose(gzfile);
-  close(file);
-
-  return rval;
-}
-#endif
-
-/* gzip the file in from_file and write it to memory to_mem, at most size bytes.
- * return the unzipped size
- */
-char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size)
-{
-  gzFile gzfile;
-  int readsize, size, alloc_size = 0;
-  char *mem = NULL;
-  const int chunk_size = 512 * 1024;
-
-  size = 0;
-
-  gzfile = BLI_gzopen(from_file, "rb");
-  for (;;) {
-    if (mem == NULL) {
-      mem = MEM_callocN(chunk_size, "BLI_ungzip_to_mem");
-      alloc_size = chunk_size;
-    }
-    else {
-      mem = MEM_reallocN(mem, size + chunk_size);
-      alloc_size += chunk_size;
-    }
-
-    readsize = gzread(gzfile, mem + size, chunk_size);
-    if (readsize > 0) {
-      size += readsize;
-    }
-    else {
+  /* Finalize the Zstd frame. */
+  size_t ret = 1;
+  while (ret != 0) {
+    ZSTD_outBuffer output = {out_buf, out_len, 0};
+    ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_end);
+    if (ZSTD_isError(ret)) {
       break;
     }
+    if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
+      break;
+    }
+    total_written += output.pos;
   }
 
-  gzclose(gzfile);
+  MEM_freeN(out_buf);
+  ZSTD_freeCCtx(ctx);
 
-  if (size == 0) {
-    MEM_freeN(mem);
-    mem = NULL;
-  }
-  else if (alloc_size != size) {
-    mem = MEM_reallocN(mem, size);
-  }
-
-  *r_size = size;
-
-  return mem;
+  return ZSTD_isError(ret) ? 0 : total_written;
 }
 
-#define CHUNK (256 * 1024)
-
-/* gzip byte array from memory and write it to file at certain position.
- * return size of gzip stream.
- */
-size_t BLI_gzip_mem_to_file_at_pos(
-    void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level)
+size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
 {
-  int ret, flush;
-  unsigned have;
-  z_stream strm;
-  unsigned char out[CHUNK];
+  fseek(file, file_offset, SEEK_SET);
 
-  BLI_fseek(file, gz_stream_offset, 0);
+  ZSTD_DCtx *ctx = ZSTD_createDCtx();
 
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  ret = deflateInit(&strm, compression_level);
-  if (ret != Z_OK) {
-    return 0;
-  }
+  size_t in_len = ZSTD_DStreamInSize();
+  void *in_buf = MEM_mallocN(in_len, __func__);
+  ZSTD_inBuffer input = {in_buf, in_len, 0};
 
-  strm.avail_in = len;
-  strm.next_in = (Bytef *)buf;
-  flush = Z_FINISH;
+  ZSTD_outBuffer output = {buf, len, 0};
 
-  do {
-    strm.avail_out = CHUNK;
-    strm.next_out = out;
-    ret = deflate(&strm, flush);
-    if (ret == Z_STREAM_ERROR) {
-      return 0;
-    }
-    have = CHUNK - strm.avail_out;
-    if (fwrite(out, 1, have, file) != have || ferror(file)) {
-      deflateEnd(&strm);
-      return 0;
-    }
-  } while (strm.avail_out == 0);
-
-  if (strm.avail_in != 0 || ret != Z_STREAM_END) {
-    return 0;
-  }
-
-  deflateEnd(&strm);
-  return (size_t)strm.total_out;
-}
-
-/* read and decompress gzip stream from file at certain position to buffer.
- * return size of decompressed data.
- */
-size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
-{
-  int ret;
-  z_stream strm;
-  size_t chunk = 256 * 1024;
-  unsigned char in[CHUNK];
-
-  BLI_fseek(file, gz_stream_offset, 0);
-
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  strm.avail_in = 0;
-  strm.next_in = Z_NULL;
-  ret = inflateInit(&strm);
-  if (ret != Z_OK) {
-    return 0;
-  }
-
-  do {
-    strm.avail_in = fread(in, 1, chunk, file);
-    strm.next_in = in;
-    if (ferror(file)) {
-      inflateEnd(&strm);
-      return 0;
+  size_t ret = 0;
+  /* Read and decompress chunks of input data until we have enough output. */
+  while (output.pos < output.size && !ZSTD_isError(ret)) {
+    input.size = fread(in_buf, 1, in_len, file);
+    if (input.size == 0) {
+      break;
     }
 
-    do {
-      strm.avail_out = len;
-      strm.next_out = (Bytef *)buf + strm.total_out;
+    /* Consume input data until we run out or have enough output. */
+    input.pos = 0;
+    while (input.pos < input.size && output.pos < output.size) {
+      ret = ZSTD_decompressStream(ctx, &output, &input);
 
-      ret = inflate(&strm, Z_NO_FLUSH);
-      if (ret == Z_STREAM_ERROR) {
-        return 0;
+      if (ZSTD_isError(ret)) {
+        break;
       }
-    } while (strm.avail_out == 0);
+    }
+  }
 
-  } while (ret != Z_STREAM_END);
+  MEM_freeN(in_buf);
+  ZSTD_freeDCtx(ctx);
 
-  inflateEnd(&strm);
-  return (size_t)strm.total_out;
+  return ZSTD_isError(ret) ? 0 : output.pos;
 }
 
-#undef CHUNK
+bool BLI_file_magic_is_gzip(const char header[4])
+{
+  /* GZIP itself starts with the magic bytes 0x1f 0x8b.
+   * The third byte indicates the compression method, which is 0x08 for DEFLATE. */
+  return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
+}
+
+bool BLI_file_magic_is_zstd(const char header[4])
+{
+  /* ZSTD files consist of concatenated frames, each either a Zstd frame or a skippable frame.
+   * Both types of frames start with a magic number: 0xFD2FB528 for Zstd frames and 0x184D2A5*
+   * for skippable frames, with the * being anything from 0 to F.
+   *
+   * To check whether a file is Zstd-compressed, we just check whether the first frame matches
+   * either. Seeking through the file until a Zstd frame is found would make things more
+   * complicated and the probability of a false positive is rather low anyways.
+   *
+   * Note that LZ4 uses a compatible format, so even though its compressed frames have a
+   * different magic number, a valid LZ4 file might also start with a skippable frame matching
+   * the second check here.
+   *
+   * For more details, see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+   */
+
+  uint32_t magic = *((uint32_t *)header);
+  if (magic == 0xFD2FB528) {
+    return true;
+  }
+  if ((magic >> 4) == 0x184D2A5) {
+    return true;
+  }
+  return false;
+}
 
 /**
  * Returns true if the file with the specified name can be written.

source/blender/blenlib/intern/filereader_file.c

@@ -0,0 +1,80 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#ifndef WIN32
+#  include <unistd.h> /* for read close */
+#else
+#  include "BLI_winstuff.h"
+#  include "winsock2.h"
+#  include <io.h> /* for open close read */
+#endif
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  int filedes;
+} RawFileReader;
+
+static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  ssize_t readsize = read(rawfile->filedes, buffer, size);
+
+  if (readsize >= 0) {
+    rawfile->reader.offset += readsize;
+  }
+
+  return readsize;
+}
+
+static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
+  return rawfile->reader.offset;
+}
+
+static void file_close(FileReader *reader)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  close(rawfile->filedes);
+  MEM_freeN(rawfile);
+}
+
+FileReader *BLI_filereader_new_file(int filedes)
+{
+  RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
+
+  rawfile->filedes = filedes;
+
+  rawfile->reader.read = file_read;
+  rawfile->reader.seek = file_seek;
+  rawfile->reader.close = file_close;
+
+  return (FileReader *)rawfile;
+}

source/blender/blenlib/intern/filereader_gzip.c

@@ -0,0 +1,108 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <zlib.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  FileReader *base;
+
+  z_stream strm;
+
+  void *in_buf;
+  size_t in_size;
+} GzipReader;
+
+static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
+{
+  GzipReader *gzip = (GzipReader *)reader;
+
+  gzip->strm.avail_out = size;
+  gzip->strm.next_out = buffer;
+
+  while (gzip->strm.avail_out > 0) {
+    if (gzip->strm.avail_in == 0) {
+      /* Ran out of buffered input data, read some more. */
+      size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
+
+      if (readsize > 0) {
+        /* We got some data, so mark the buffer as refilled. */
+        gzip->strm.avail_in = readsize;
+        gzip->strm.next_in = gzip->in_buf;
+      }
+      else {
+        /* The underlying file is EOF, so return as much as we can. */
+        break;
+      }
+    }
+
+    int ret = inflate(&gzip->strm, Z_NO_FLUSH);
+
+    if (ret != Z_OK && ret != Z_BUF_ERROR) {
+      break;
+    }
+  }
+
+  ssize_t read_len = size - gzip->strm.avail_out;
+  gzip->reader.offset += read_len;
+  return read_len;
+}
+
+static void gzip_close(FileReader *reader)
+{
+  GzipReader *gzip = (GzipReader *)reader;
+
+  if (inflateEnd(&gzip->strm) != Z_OK) {
+    printf("close gzip stream error\n");
+  }
+  MEM_freeN((void *)gzip->in_buf);
+
+  gzip->base->close(gzip->base);
+  MEM_freeN(gzip);
+}
+
+FileReader *BLI_filereader_new_gzip(FileReader *base)
+{
+  GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__);
+  gzip->base = base;
+
+  if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) {
+    MEM_freeN(gzip);
+    return NULL;
+  }
+
+  gzip->in_size = 256 * 2014;
+  gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf");
+
+  gzip->reader.read = gzip_read;
+  gzip->reader.seek = NULL;
+  gzip->reader.close = gzip_close;
+
+  return (FileReader *)gzip;
+}

source/blender/blenlib/intern/filereader_memory.c

@@ -0,0 +1,145 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+#include "BLI_mmap.h"
+
+#include "MEM_guardedalloc.h"
+
+/* This file implements both memory-backed and memory-mapped-file-backed reading. */
+typedef struct {
+  FileReader reader;
+
+  const char *data;
+  BLI_mmap_file *mmap;
+  size_t length;
+} MemoryReader;
+
+static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  /* Don't read more bytes than there are available in the buffer. */
+  size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
+
+  memcpy(buffer, mem->data + mem->reader.offset, readsize);
+  mem->reader.offset += readsize;
+
+  return readsize;
+}
+
+static off64_t memory_seek(FileReader *reader, off64_t offset, int whence)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  off64_t new_pos;
+  if (whence == SEEK_CUR) {
+    new_pos = mem->reader.offset + offset;
+  }
+  else if (whence == SEEK_SET) {
+    new_pos = offset;
+  }
+  else if (whence == SEEK_END) {
+    new_pos = mem->length + offset;
+  }
+  else {
+    return -1;
+  }
+
+  if (new_pos < 0 || new_pos > mem->length) {
+    return -1;
+  }
+
+  mem->reader.offset = new_pos;
+  return mem->reader.offset;
+}
+
+static void memory_close_raw(FileReader *reader)
+{
+  MEM_freeN(reader);
+}
+
+FileReader *BLI_filereader_new_memory(const void *data, size_t len)
+{
+  MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
+
+  mem->data = (const char *)data;
+  mem->length = len;
+
+  mem->reader.read = memory_read_raw;
+  mem->reader.seek = memory_seek;
+  mem->reader.close = memory_close_raw;
+
+  return (FileReader *)mem;
+}
+
+/* Memory-mapped file reading.
+ * By using `mmap()`, we can map a file so that it can be treated like normal memory,
+ * meaning that we can just read from it with `memcpy()` etc.
+ * This avoids system call overhead and can significantly speed up file loading.
+ */
+
+static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  /* Don't read more bytes than there are available in the buffer. */
+  size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
+
+  if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) {
+    return 0;
+  }
+
+  mem->reader.offset += readsize;
+
+  return readsize;
+}
+
+static void memory_close_mmap(FileReader *reader)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+  BLI_mmap_free(mem->mmap);
+  MEM_freeN(mem);
+}
+
+FileReader *BLI_filereader_new_mmap(int filedes)
+{
+  BLI_mmap_file *mmap = BLI_mmap_open(filedes);
+  if (mmap == NULL) {
+    return NULL;
+  }
+
+  MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
+
+  mem->mmap = mmap;
+  mem->length = BLI_lseek(filedes, 0, SEEK_END);
+
+  mem->reader.read = memory_read_mmap;
+  mem->reader.seek = memory_seek;
+  mem->reader.close = memory_close_mmap;
+
+  return (FileReader *)mem;
+}

source/blender/blenlib/intern/filereader_zstd.c

@@ -0,0 +1,335 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+#include <zstd.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_endian_switch.h"
+#include "BLI_filereader.h"
+#include "BLI_math_base.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  FileReader *base;
+
+  ZSTD_DCtx *ctx;
+  ZSTD_inBuffer in_buf;
+  size_t in_buf_max_size;
+
+  struct {
+    int num_frames;
+    size_t *compressed_ofs;
+    size_t *uncompressed_ofs;
+
+    char *cached_content;
+    int cached_frame;
+  } seek;
+} ZstdReader;
+
+static bool zstd_read_u32(FileReader *base, uint32_t *val)
+{
+  if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
+    return false;
+  }
+#ifdef __BIG_ENDIAN__
+  BLI_endian_switch_uint32(val);
+#endif
+  return true;
+}
+
+static bool zstd_read_seek_table(ZstdReader *zstd)
+{
+  FileReader *base = zstd->base;
+
+  /* The seek table frame is at the end of the file, so seek there
+   * and verify that there is enough data. */
+  if (base->seek(base, -4, SEEK_END) < 13) {
+    return false;
+  }
+  uint32_t magic;
+  if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
+    return false;
+  }
+
+  uint8_t flags;
+  if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
+    return false;
+  }
+  /* Bit 7 indicates checksums. Bits 5 and 6 must be zero. */
+  bool has_checksums = (flags & 0x80);
+  if (flags & 0x60) {
+    return false;
+  }
+
+  uint32_t num_frames;
+  if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &num_frames)) {
+    return false;
+  }
+
+  /* Each frame has either 2 or 3 uint32_t, and after that we have
+   * num_frames, flags and magic for another 9 bytes. */
+  uint32_t expected_frame_length = num_frames * (has_checksums ? 12 : 8) + 9;
+  /* The frame starts with another magic number and its length, but these
+   * two fields are not included when counting length. */
+  off64_t frame_start_ofs = 8 + expected_frame_length;
+  /* Sanity check: Before the start of the seek table frame,
+   * there must be num_frames frames, each of which at least 8 bytes long. */
+  off64_t seek_frame_start = base->seek(base, -frame_start_ofs, SEEK_END);
+  if (seek_frame_start < num_frames * 8) {
+    return false;
+  }
+
+  if (!zstd_read_u32(base, &magic) || magic != 0x184D2A5E) {
+    return false;
+  }
+
+  uint32_t frame_length;
+  if (!zstd_read_u32(base, &frame_length) || frame_length != expected_frame_length) {
+    return false;
+  }
+
+  zstd->seek.num_frames = num_frames;
+  zstd->seek.compressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__);
+  zstd->seek.uncompressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__);
+
+  size_t compressed_ofs = 0;
+  size_t uncompressed_ofs = 0;
+  for (int i = 0; i < num_frames; i++) {
+    uint32_t compressed_size, uncompressed_size;
+    if (!zstd_read_u32(base, &compressed_size) || !zstd_read_u32(base, &uncompressed_size)) {
+      break;
+    }
+    if (has_checksums && base->seek(base, 4, SEEK_CUR) < 0) {
+      break;
+    }
+    zstd->seek.compressed_ofs[i] = compressed_ofs;
+    zstd->seek.uncompressed_ofs[i] = uncompressed_ofs;
+    compressed_ofs += compressed_size;
+    uncompressed_ofs += uncompressed_size;
+  }
+  zstd->seek.compressed_ofs[num_frames] = compressed_ofs;
+  zstd->seek.uncompressed_ofs[num_frames] = uncompressed_ofs;
+
+  /* Seek to the end of the previous frame for the following BHead frame detection. */
+  if (seek_frame_start != compressed_ofs || base->seek(base, seek_frame_start, SEEK_SET) < 0) {
+    MEM_freeN(zstd->seek.compressed_ofs);
+    MEM_freeN(zstd->seek.uncompressed_ofs);
+    memset(&zstd->seek, 0, sizeof(zstd->seek));
+    return false;
+  }
+
+  zstd->seek.cached_frame = -1;
+
+  return true;
+}
+
+/* Find out which frame contains the given position in the uncompressed stream.
+ * Basically just bisection. */
+static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
+{
+  int low = 0, high = zstd->seek.num_frames;
+
+  if (pos >= zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) {
+    return -1;
+  }
+
+  while (low + 1 < high) {
+    int mid = low + ((high - low) >> 1);
+    if (zstd->seek.uncompressed_ofs[mid] <= pos) {
+      low = mid;
+    }
+    else {
+      high = mid;
+    }
+  }
+
+  return low;
+}
+
+/* Ensure that the currently loaded frame is the correct one. */
+static const char *zstd_ensure_cache(ZstdReader *zstd, int frame)
+{
+  if (zstd->seek.cached_frame == frame) {
+    /* Cached frame matches, so just return it. */
+    return zstd->seek.cached_content;
+  }
+
+  /* Cached frame doesn't match, so discard it and cache the wanted one onstead. */
+  MEM_SAFE_FREE(zstd->seek.cached_content);
+
+  size_t compressed_size = zstd->seek.compressed_ofs[frame + 1] - zstd->seek.compressed_ofs[frame];
+  size_t uncompressed_size = zstd->seek.uncompressed_ofs[frame + 1] -
+                             zstd->seek.uncompressed_ofs[frame];
+
+  char *uncompressed_data = MEM_mallocN(uncompressed_size, __func__);
+  char *compressed_data = MEM_mallocN(compressed_size, __func__);
+  if (zstd->base->seek(zstd->base, zstd->seek.compressed_ofs[frame], SEEK_SET) < 0 ||
+      zstd->base->read(zstd->base, compressed_data, compressed_size) < compressed_size) {
+    MEM_freeN(compressed_data);
+    MEM_freeN(uncompressed_data);
+    return NULL;
+  }
+
+  size_t res = ZSTD_decompressDCtx(
+      zstd->ctx, uncompressed_data, uncompressed_size, compressed_data, compressed_size);
+  MEM_freeN(compressed_data);
+  if (ZSTD_isError(res) || res < uncompressed_size) {
+    MEM_freeN(uncompressed_data);
+    return NULL;
+  }
+
+  zstd->seek.cached_frame = frame;
+  zstd->seek.cached_content = uncompressed_data;
+  return uncompressed_data;
+}
+
+static ssize_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+
+  size_t end_offset = zstd->reader.offset + size, read_len = 0;
+  while (zstd->reader.offset < end_offset) {
+    int frame = zstd_frame_from_pos(zstd, zstd->reader.offset);
+    if (frame < 0) {
+      /* EOF is reached, so return as much as we can. */
+      break;
+    }
+
+    const char *framedata = zstd_ensure_cache(zstd, frame);
+    if (framedata == NULL) {
+      /* Error while reading the frame, so return as much as we can. */
+      break;
+    }
+
+    size_t frame_end_offset = min_zz(zstd->seek.uncompressed_ofs[frame + 1], end_offset);
+    size_t frame_read_len = frame_end_offset - zstd->reader.offset;
+
+    size_t offset_in_frame = zstd->reader.offset - zstd->seek.uncompressed_ofs[frame];
+    memcpy((char *)buffer + read_len, framedata + offset_in_frame, frame_read_len);
+    read_len += frame_read_len;
+    zstd->reader.offset = frame_end_offset;
+  }
+
+  return read_len;
+}
+
+static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+  off64_t new_pos;
+  if (whence == SEEK_SET) {
+    new_pos = offset;
+  }
+  else if (whence == SEEK_END) {
+    new_pos = zstd->seek.uncompressed_ofs[zstd->seek.num_frames] + offset;
+  }
+  else {
+    new_pos = zstd->reader.offset + offset;
+  }
+
+  if (new_pos < 0 || new_pos > zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) {
+    return -1;
+  }
+  zstd->reader.offset = new_pos;
+  return zstd->reader.offset;
+}
+
+static ssize_t zstd_read(FileReader *reader, void *buffer, size_t size)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+  ZSTD_outBuffer output = {buffer, size, 0};
+
+  while (output.pos < output.size) {
+    if (zstd->in_buf.pos == zstd->in_buf.size) {
+      /* Ran out of buffered input data, read some more. */
+      zstd->in_buf.pos = 0;
+      ssize_t readsize = zstd->base->read(
+          zstd->base, (char *)zstd->in_buf.src, zstd->in_buf_max_size);
+
+      if (readsize > 0) {
+        /* We got some data, so mark the buffer as refilled. */
+        zstd->in_buf.size = readsize;
+      }
+      else {
+        /* The underlying file is EOF, so return as much as we can. */
+        break;
+      }
+    }
+
+    if (ZSTD_isError(ZSTD_decompressStream(zstd->ctx, &output, &zstd->in_buf))) {
+      break;
+    }
+  }
+
+  zstd->reader.offset += output.pos;
+  return output.pos;
+}
+
+static void zstd_close(FileReader *reader)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+
+  ZSTD_freeDCtx(zstd->ctx);
+  if (zstd->reader.seek) {
+    MEM_freeN(zstd->seek.uncompressed_ofs);
+    MEM_freeN(zstd->seek.compressed_ofs);
+    MEM_freeN(zstd->seek.cached_content);
+  }
+  else {
+    MEM_freeN((void *)zstd->in_buf.src);
+  }
+
+  zstd->base->close(zstd->base);
+  MEM_freeN(zstd);
+}
+
+FileReader *BLI_filereader_new_zstd(FileReader *base)
+{
+  ZstdReader *zstd = MEM_callocN(sizeof(ZstdReader), __func__);
+
+  zstd->ctx = ZSTD_createDCtx();
+  zstd->base = base;
+
+  if (zstd_read_seek_table(zstd)) {
+    zstd->reader.read = zstd_read_seekable;
+    zstd->reader.seek = zstd_seek;
+  }
+  else {
+    zstd->reader.read = zstd_read;
+    zstd->reader.seek = NULL;
+
+    zstd->in_buf_max_size = ZSTD_DStreamInSize();
+    zstd->in_buf.src = MEM_mallocN(zstd->in_buf_max_size, "zstd in buf");
+    zstd->in_buf.size = zstd->in_buf_max_size;
+    /* This signals that the buffer has run out,
+     * which will make the read function refill it on the first call. */
+    zstd->in_buf.pos = zstd->in_buf_max_size;
+  }
+  zstd->reader.close = zstd_close;
+
+  return (FileReader *)zstd;
+}

source/blender/blenlib/intern/freetypefont.c

@@ -302,7 +302,7 @@ static VFontData *objfnt_to_ftvfontdata(PackedFile *pf)
   /* Get the name. */
   if (face->family_name) {
     BLI_snprintf(vfd->name, sizeof(vfd->name), "%s %s", face->family_name, face->style_name);
-    BLI_utf8_invalid_strip(vfd->name, strlen(vfd->name));
+    BLI_str_utf8_invalid_strip(vfd->name, strlen(vfd->name));
   }
 
   /* Select a character map. */

source/blender/blenlib/intern/math_base_inline.c

@@ -363,6 +363,14 @@ MINLINE signed char round_db_to_char_clamp(double a){
 #undef _round_clamp_fl_impl
 #undef _round_clamp_db_impl
 
+/**
+ * Round to closest even number, halfway cases are rounded away from zero.
+ */
+MINLINE float round_to_even(float f)
+{
+  return roundf(f * 0.5f) * 2.0f;
+}
+
 /* integer division that rounds 0.5 up, particularly useful for color blending
  * with integers, to avoid gradual darkening when rounding down */
 MINLINE int divide_round_i(int a, int b)

source/blender/blenlib/intern/string_utf8.c

@@ -70,7 +70,7 @@ static const size_t utf8_skip_data[256] = {
  *
  * \return the offset of the first invalid byte.
  */
-ptrdiff_t BLI_utf8_invalid_byte(const char *str, size_t length)
+ptrdiff_t BLI_str_utf8_invalid_byte(const char *str, size_t length)
 {
   const unsigned char *p, *perr, *pend = (const unsigned char *)str + length;
   unsigned char c;
@@ -200,14 +200,14 @@ utf8_error:
  *
  * \return number of stripped bytes.
  */
-int BLI_utf8_invalid_strip(char *str, size_t length)
+int BLI_str_utf8_invalid_strip(char *str, size_t length)
 {
   ptrdiff_t bad_char;
   int tot = 0;
 
   BLI_assert(str[length] == '\0');
 
-  while ((bad_char = BLI_utf8_invalid_byte(str, length)) != -1) {
+  while ((bad_char = BLI_str_utf8_invalid_byte(str, length)) != -1) {
     str += bad_char;
     length -= (size_t)(bad_char + 1);
 

source/blender/blenlib/tests/BLI_string_utf8_test.cc

@@ -266,7 +266,7 @@ static const char *utf8_invalid_tests[][3] = {
 };
 /* clang-format on */
 
-/* BLI_utf8_invalid_strip (and indirectly, BLI_utf8_invalid_byte). */
+/* BLI_str_utf8_invalid_strip (and indirectly, BLI_str_utf8_invalid_byte). */
 TEST(string, Utf8InvalidBytes)
 {
   for (int i = 0; utf8_invalid_tests[i][0] != nullptr; i++) {
@@ -277,7 +277,7 @@ TEST(string, Utf8InvalidBytes)
     char buff[80];
     memcpy(buff, tst, sizeof(buff));
 
-    const int num_errors_found = BLI_utf8_invalid_strip(buff, sizeof(buff) - 1);
+    const int num_errors_found = BLI_str_utf8_invalid_strip(buff, sizeof(buff) - 1);
 
     printf("[%02d] -> [%02d] \"%s\"  ->  \"%s\"\n", num_errors, num_errors_found, tst, buff);
     EXPECT_EQ(num_errors_found, num_errors);

source/blender/blenloader/BLO_undofile.h

@@ -24,6 +24,8 @@
  * \ingroup blenloader
  */
 
+#include "BLI_filereader.h"
+
 struct GHash;
 struct Scene;
 
@@ -65,6 +67,16 @@ typedef struct MemFileUndoData {
   size_t undo_size;
 } MemFileUndoData;
 
+/* FileReader-compatible wrapper for reading MemFiles */
+typedef struct {
+  FileReader reader;
+
+  MemFile *memfile;
+  int undo_direction;
+
+  bool memchunk_identical;
+} UndoReader;
+
 /* actually only used writefile.c */
 
 void BLO_memfile_write_init(MemFileWriteData *mem_data,
@@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile,
                                          struct Main *bmain,
                                          struct Scene **r_scene);
 extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename);
+
+FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction);

source/blender/blenloader/CMakeLists.txt

@@ -42,7 +42,7 @@ set(INC
 )
 
 set(INC_SYS
-  ${ZLIB_INCLUDE_DIRS}
+  ${ZSTD_INCLUDE_DIRS}
 )
 
 set(SRC

source/blender/blenloader/intern/readfile.c

@@ -21,8 +21,6 @@
  * \ingroup blenloader
  */
 
-#include "zlib.h"
-
 #include <ctype.h> /* for isdigit. */
 #include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
 #include <limits.h>
@@ -71,7 +69,6 @@
 #include "BLI_math.h"
 #include "BLI_memarena.h"
 #include "BLI_mempool.h"
-#include "BLI_mmap.h"
 #include "BLI_threads.h"
 
 #include "PIL_time.h"
@@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd)
        */
       if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
         bhead4.code = DATA;
-        readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL);
+        readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4));
 
         if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
           if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd)
       }
       else {
         bhead8.code = DATA;
-        readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL);
+        readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8));
 
         if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
           if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd)
         /* pass */
       }
 #ifdef USE_BHEAD_READ_ON_DEMAND
-      else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
+      else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
         /* Delay reading bhead content. */
         new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead");
         if (new_bhead) {
           new_bhead->next = new_bhead->prev = NULL;
-          new_bhead->file_offset = fd->file_offset;
+          new_bhead->file_offset = fd->file->offset;
           new_bhead->has_data = false;
           new_bhead->is_memchunk_identical = false;
           new_bhead->bhead = bhead;
-          off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR);
+          off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR);
           if (seek_new == -1) {
             fd->is_eof = true;
             MEM_freeN(new_bhead);
             new_bhead = NULL;
           }
-          BLI_assert(fd->file_offset == seek_new);
+          BLI_assert(fd->file->offset == seek_new);
         }
         else {
           fd->is_eof = true;
@@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd)
           new_bhead->is_memchunk_identical = false;
           new_bhead->bhead = bhead;
 
-          readsize = fd->read(
-              fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical);
+          readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len);
 
-          if (readsize != (ssize_t)bhead.len) {
+          if (readsize != bhead.len) {
             fd->is_eof = true;
             MEM_freeN(new_bhead);
             new_bhead = NULL;
           }
+
+          if (fd->flags & FD_FLAGS_IS_MEMFILE) {
+            new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
+          }
         }
         else {
           fd->is_eof = true;
@@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
   bool success = true;
   BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
   BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0);
-  off64_t offset_backup = fd->file_offset;
-  if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) {
+  off64_t offset_backup = fd->file->offset;
+  if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) {
     success = false;
   }
   else {
-    if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
-        (ssize_t)new_bhead->bhead.len) {
+    if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) {
       success = false;
     }
+    if (fd->flags & FD_FLAGS_IS_MEMFILE) {
+      new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
+    }
   }
-  if (fd->seek(fd, offset_backup, SEEK_SET) == -1) {
+  if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) {
     success = false;
   }
   return success;
@@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd)
   ssize_t readsize;
 
   /* read in the header data */
-  readsize = fd->read(fd, header, sizeof(header), NULL);
+  readsize = fd->file->read(fd->file, header, sizeof(header));
 
   if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') &&
       ELEM(header[8], 'v', 'V') &&
@@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd)
 
 /** \} */
 
-/* -------------------------------------------------------------------- */
-/** \name File Data API
- * \{ */
-
-/* Regular file reading. */
-
-static ssize_t fd_read_data_from_file(FileData *filedata,
-                                      void *buffer,
-                                      size_t size,
-                                      bool *UNUSED(r_is_memchunck_identical))
-{
-  ssize_t readsize = read(filedata->filedes, buffer, size);
-
-  if (readsize < 0) {
-    readsize = EOF;
-  }
-  else {
-    filedata->file_offset += readsize;
-  }
-
-  return readsize;
-}
-
-static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence)
-{
-  filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence);
-  return filedata->file_offset;
-}
-
-/* GZip file reading. */
-
-static ssize_t fd_read_gzip_from_file(FileData *filedata,
-                                      void *buffer,
-                                      size_t size,
-                                      bool *UNUSED(r_is_memchunck_identical))
-{
-  BLI_assert(size <= INT_MAX);
-
-  ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size);
-
-  if (readsize < 0) {
-    readsize = EOF;
-  }
-  else {
-    filedata->file_offset += readsize;
-  }
-
-  return readsize;
-}
-
-/* Memory reading. */
-
-static ssize_t fd_read_from_memory(FileData *filedata,
-                                   void *buffer,
-                                   size_t size,
-                                   bool *UNUSED(r_is_memchunck_identical))
-{
-  /* don't read more bytes than there are available in the buffer */
-  ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset);
-
-  memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize);
-  filedata->file_offset += readsize;
-
-  return readsize;
-}
-
-/* Memory-mapped file reading.
- * By using mmap(), we can map a file so that it can be treated like normal memory,
- * meaning that we can just read from it with memcpy() etc.
- * This avoids system call overhead and can significantly speed up file loading.
- */
-
-static ssize_t fd_read_from_mmap(FileData *filedata,
-                                 void *buffer,
-                                 size_t size,
-                                 bool *UNUSED(r_is_memchunck_identical))
-{
-  /* don't read more bytes than there are available in the buffer */
-  size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset));
-
-  if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) {
-    return 0;
-  }
-
-  filedata->file_offset += readsize;
-
-  return readsize;
-}
-
-static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence)
-{
-  off64_t new_pos;
-  if (whence == SEEK_CUR) {
-    new_pos = filedata->file_offset + offset;
-  }
-  else if (whence == SEEK_SET) {
-    new_pos = offset;
-  }
-  else if (whence == SEEK_END) {
-    new_pos = filedata->buffersize + offset;
-  }
-  else {
-    return -1;
-  }
-
-  if (new_pos < 0 || new_pos > filedata->buffersize) {
-    return -1;
-  }
-
-  filedata->file_offset = new_pos;
-  return filedata->file_offset;
-}
-
-/* MemFile reading. */
-
-static ssize_t fd_read_from_memfile(FileData *filedata,
-                                    void *buffer,
-                                    size_t size,
-                                    bool *r_is_memchunck_identical)
-{
-  static size_t seek = SIZE_MAX; /* the current position */
-  static size_t offset = 0;      /* size of previous chunks */
-  static MemFileChunk *chunk = NULL;
-  size_t chunkoffset, readsize, totread;
-
-  if (r_is_memchunck_identical != NULL) {
-    *r_is_memchunck_identical = true;
-  }
-
-  if (size == 0) {
-    return 0;
-  }
-
-  if (seek != (size_t)filedata->file_offset) {
-    chunk = filedata->memfile->chunks.first;
-    seek = 0;
-
-    while (chunk) {
-      if (seek + chunk->size > (size_t)filedata->file_offset) {
-        break;
-      }
-      seek += chunk->size;
-      chunk = chunk->next;
-    }
-    offset = seek;
-    seek = (size_t)filedata->file_offset;
-  }
-
-  if (chunk) {
-    totread = 0;
-
-    do {
-      /* first check if it's on the end if current chunk */
-      if (seek - offset == chunk->size) {
-        offset += chunk->size;
-        chunk = chunk->next;
-      }
-
-      /* debug, should never happen */
-      if (chunk == NULL) {
-        CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk");
-        return 0;
-      }
-
-      chunkoffset = seek - offset;
-      readsize = size - totread;
-
-      /* data can be spread over multiple chunks, so clamp size
-       * to within this chunk, and then it will read further in
-       * the next chunk */
-      if (chunkoffset + readsize > chunk->size) {
-        readsize = chunk->size - chunkoffset;
-      }
-
-      memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
-      totread += readsize;
-      filedata->file_offset += readsize;
-      seek += readsize;
-      if (r_is_memchunck_identical != NULL) {
-        /* `is_identical` of current chunk represents whether it changed compared to previous undo
-         * step. this is fine in redo case, but not in undo case, where we need an extra flag
-         * defined when saving the next (future) step after the one we want to restore, as we are
-         * supposed to 'come from' that future undo step, and not the one before current one. */
-        *r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ?
-                                         chunk->is_identical :
-                                         chunk->is_identical_future;
-      }
-    } while (totread < size);
-
-    return (ssize_t)totread;
-  }
-
-  return 0;
-}
-
 static FileData *filedata_new(BlendFileReadReport *reports)
 {
   BLI_assert(reports != NULL);
 
   FileData *fd = MEM_callocN(sizeof(FileData), "FileData");
 
-  fd->filedes = -1;
-  fd->gzfiledes = NULL;
-
   fd->memsdna = DNA_sdna_current_get();
 
   fd->datamap = oldnewmap_new();
@@ -1387,78 +1191,66 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports)
 
 static FileData *blo_filedata_from_file_descriptor(const char *filepath,
                                                    BlendFileReadReport *reports,
-                                                   int file)
+                                                   int filedes)
 {
-  FileDataReadFn *read_fn = NULL;
-  FileDataSeekFn *seek_fn = NULL; /* Optional. */
-  size_t buffersize = 0;
-  BLI_mmap_file *mmap_file = NULL;
-
-  gzFile gzfile = (gzFile)Z_NULL;
-
   char header[7];
+  FileReader *rawfile = BLI_filereader_new_file(filedes);
+  FileReader *file = NULL;
 
-  /* Regular file. */
   errno = 0;
-  if (read(file, header, sizeof(header)) != sizeof(header)) {
+  /* If opening the file failed or we can't read the header, give up. */
+  if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
     BKE_reportf(reports->reports,
                 RPT_WARNING,
                 "Unable to read '%s': %s",
                 filepath,
                 errno ? strerror(errno) : TIP_("insufficient content"));
+    if (rawfile) {
+      rawfile->close(rawfile);
+    }
+    else {
+      close(filedes);
+    }
     return NULL;
   }
 
-  /* Regular file. */
+  /* Rewind the file after reading the header. */
+  rawfile->seek(rawfile, 0, SEEK_SET);
+
+  /* Check if we have a regular file. */
   if (memcmp(header, "BLENDER", sizeof(header)) == 0) {
-    read_fn = fd_read_data_from_file;
-    seek_fn = fd_seek_data_from_file;
-
-    mmap_file = BLI_mmap_open(file);
-    if (mmap_file != NULL) {
-      read_fn = fd_read_from_mmap;
-      seek_fn = fd_seek_from_mmap;
-      buffersize = BLI_lseek(file, 0, SEEK_END);
+    /* Try opening the file with memory-mapped IO. */
+    file = BLI_filereader_new_mmap(filedes);
+    if (file == NULL) {
+      /* mmap failed, so just keep using rawfile. */
+      file = rawfile;
+      rawfile = NULL;
+    }
+  }
+  else if (BLI_file_magic_is_gzip(header)) {
+    file = BLI_filereader_new_gzip(rawfile);
+    if (file != NULL) {
+      rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */
+    }
+  }
+  else if (BLI_file_magic_is_zstd(header)) {
+    file = BLI_filereader_new_zstd(rawfile);
+    if (file != NULL) {
+      rawfile = NULL; /* The Zstd FileReader takes ownership of `rawfile`. */
     }
   }
 
-  BLI_lseek(file, 0, SEEK_SET);
-
-  /* Gzip file. */
-  errno = 0;
-  if ((read_fn == NULL) &&
-      /* Check header magic. */
-      (header[0] == 0x1f && header[1] == 0x8b)) {
-    gzfile = BLI_gzopen(filepath, "rb");
-    if (gzfile == (gzFile)Z_NULL) {
-      BKE_reportf(reports->reports,
-                  RPT_WARNING,
-                  "Unable to open '%s': %s",
-                  filepath,
-                  errno ? strerror(errno) : TIP_("unknown error reading file"));
-      return NULL;
-    }
-
-    /* 'seek_fn' is too slow for gzip, don't set it. */
-    read_fn = fd_read_gzip_from_file;
-    /* Caller must close. */
-    file = -1;
+  /* Clean up `rawfile` if it wasn't taken over. */
+  if (rawfile != NULL) {
+    rawfile->close(rawfile);
   }
-
-  if (read_fn == NULL) {
+  if (file == NULL) {
     BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath);
     return NULL;
   }
 
   FileData *fd = filedata_new(reports);
-
-  fd->filedes = file;
-  fd->gzfiledes = gzfile;
-
-  fd->read = read_fn;
-  fd->seek = seek_fn;
-  fd->mmap_file = mmap_file;
-  fd->buffersize = buffersize;
+  fd->file = file;
 
   return fd;
 }
@@ -1475,11 +1267,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead
                 errno ? strerror(errno) : TIP_("unknown error reading file"));
     return NULL;
   }
-  FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file);
-  if ((fd == NULL) || (fd->filedes == -1)) {
-    close(file);
-  }
-  return fd;
+  return blo_filedata_from_file_descriptor(filepath, reports, file);
 }
 
 /* cannot be called with relative paths anymore! */
@@ -1513,50 +1301,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
   return NULL;
 }
 
-static ssize_t fd_read_gzip_from_memory(FileData *filedata,
-                                        void *buffer,
-                                        size_t size,
-                                        bool *UNUSED(r_is_memchunck_identical))
-{
-  int err;
-
-  filedata->strm.next_out = (Bytef *)buffer;
-  filedata->strm.avail_out = (uint)size;
-
-  /* Inflate another chunk. */
-  err = inflate(&filedata->strm, Z_SYNC_FLUSH);
-
-  if (err == Z_STREAM_END) {
-    return 0;
-  }
-  if (err != Z_OK) {
-    CLOG_ERROR(&LOG, "ZLib error (code %d)", err);
-    return 0;
-  }
-
-  filedata->file_offset += size;
-
-  return (ssize_t)size;
-}
-
-static int fd_read_gzip_from_memory_init(FileData *fd)
-{
-
-  fd->strm.next_in = (Bytef *)fd->buffer;
-  fd->strm.avail_in = fd->buffersize;
-  fd->strm.total_out = 0;
-  fd->strm.zalloc = Z_NULL;
-  fd->strm.zfree = Z_NULL;
-
-  if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) {
-    return 0;
-  }
-
-  fd->read = fd_read_gzip_from_memory;
-
-  return 1;
-}
-
 FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports)
 {
   if (!mem || memsize < SIZEOFBLENDERHEADER) {
@@ -1565,24 +1309,24 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe
     return NULL;
   }
 
+  FileReader *mem_file = BLI_filereader_new_memory(mem, memsize);
+  FileReader *file = mem_file;
+
+  if (BLI_file_magic_is_gzip(mem)) {
+    file = BLI_filereader_new_gzip(mem_file);
+  }
+  else if (BLI_file_magic_is_zstd(mem)) {
+    file = BLI_filereader_new_zstd(mem_file);
+  }
+
+  if (file == NULL) {
+    /* Compression initialization failed. */
+    mem_file->close(mem_file);
+    return NULL;
+  }
+
   FileData *fd = filedata_new(reports);
-  const char *cp = mem;
-
-  fd->buffer = mem;
-  fd->buffersize = memsize;
-
-  /* test if gzip */
-  if (cp[0] == 0x1f && cp[1] == 0x8b) {
-    if (0 == fd_read_gzip_from_memory_init(fd)) {
-      blo_filedata_free(fd);
-      return NULL;
-    }
-  }
-  else {
-    fd->read = fd_read_from_memory;
-  }
-
-  fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
+  fd->file = file;
 
   return blo_decode_and_check(fd, reports->reports);
 }
@@ -1597,11 +1341,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
   }
 
   FileData *fd = filedata_new(reports);
-  fd->memfile = memfile;
+  fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction);
   fd->undo_direction = params->undo_direction;
-
-  fd->read = fd_read_from_memfile;
-  fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
+  fd->flags |= FD_FLAGS_IS_MEMFILE;
 
   return blo_decode_and_check(fd, reports->reports);
 }
@@ -1609,30 +1351,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
 void blo_filedata_free(FileData *fd)
 {
   if (fd) {
-    if (fd->filedes != -1) {
-      close(fd->filedes);
-    }
-
-    if (fd->gzfiledes != NULL) {
-      gzclose(fd->gzfiledes);
-    }
-
-    if (fd->strm.next_in) {
-      int err = inflateEnd(&fd->strm);
-      if (err != Z_OK) {
-        CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err);
-      }
-    }
-
-    if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
-      MEM_freeN((void *)fd->buffer);
-      fd->buffer = NULL;
-    }
-
-    if (fd->mmap_file) {
-      BLI_mmap_free(fd->mmap_file);
-      fd->mmap_file = NULL;
-    }
+    fd->file->close(fd->file);
 
     /* Free all BHeadN data blocks */
 #ifndef NDEBUG
@@ -1640,7 +1359,7 @@ void blo_filedata_free(FileData *fd)
 #else
     /* Sanity check we're not keeping memory we don't need. */
     LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) {
-      if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
+      if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
         BLI_assert(new_bhead->has_data == 0);
       }
       MEM_freeN(new_bhead);
@@ -2096,7 +1815,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id),
 
 void blo_cache_storage_init(FileData *fd, Main *bmain)
 {
-  if (fd->memfile != NULL) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     BLI_assert(fd->cache_storage == NULL);
     fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__);
     fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
@@ -2261,7 +1980,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
  * undo since DNA must match. */
 static const void *peek_struct_undo(FileData *fd, BHead *bhead)
 {
-  BLI_assert(fd->memfile != NULL);
+  BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE);
   UNUSED_VARS_NDEBUG(fd);
   return (bhead->len) ? (const void *)(bhead + 1) : NULL;
 }
@@ -3679,7 +3398,7 @@ static BHead *read_libblock(FileData *fd,
    * When datablocks are changed but still exist, we restore them at the old
    * address and inherit recalc flags for the dependency graph. */
   ID *id_old = NULL;
-  if (fd->memfile != NULL) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) {
       if (r_id) {
         *r_id = id_old;
@@ -3980,13 +3699,14 @@ static void lib_link_all(FileData *fd, Main *bmain)
       continue;
     }
 
-    if (fd->memfile != NULL && GS(id->name) == ID_WM) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) {
       /* No load UI for undo memfiles.
        * Only WM currently, SCR needs it still (see below), and so does WS? */
       continue;
     }
 
-    if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo &&
+        (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
       /* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across
        * current undo step, and old IDs re-use their old memory address, we do not need to liblink
        * it at all. */
@@ -4165,7 +3885,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
   BlendFileData *bfd;
   ListBase mainlist = {NULL, NULL};
 
-  if (fd->memfile != NULL) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step");
   }
 
@@ -4256,7 +3976,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
   }
 
   /* do before read_libraries, but skip undo case */
-  if (fd->memfile == NULL) {
+  if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
     if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
       do_versions(fd, NULL, bfd->main);
     }
@@ -4278,7 +3998,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
     fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries;
 
     /* Skip in undo case. */
-    if (fd->memfile == NULL) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
       /* Note that we can't recompute user-counts at this point in undo case, we play too much with
        * IDs from different memory realms, and Main database is not in a fully valid state yet.
        */
@@ -4311,7 +4031,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
 
     /* Now that all our data-blocks are loaded,
      * we can re-generate overrides from their references. */
-    if (fd->memfile == NULL) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
       /* Do not apply in undo case! */
       fd->reports->duration.lib_overrides = PIL_check_seconds_timer();
 
@@ -4391,7 +4111,7 @@ static void sort_bhead_old_map(FileData *fd)
 static BHead *find_previous_lib(FileData *fd, BHead *bhead)
 {
   /* Skip library data-blocks in undo, see comment in read_libblock. */
-  if (fd->memfile) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     return NULL;
   }
 
@@ -5850,7 +5570,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
 
 bool BLO_read_data_is_undo(BlendDataReader *reader)
 {
-  return reader->fd->memfile != NULL;
+  return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
 }
 
 void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr)
@@ -5870,7 +5590,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader)
 
 bool BLO_read_lib_is_undo(BlendLibReader *reader)
 {
-  return reader->fd->memfile != NULL;
+  return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
 }
 
 Main *BLO_read_lib_get_main(BlendLibReader *reader)

source/blender/blenloader/intern/readfile.h

@@ -28,10 +28,10 @@
 #  include "BLI_winstuff.h"
 #endif
 
+#include "BLI_filereader.h"
 #include "DNA_sdna_types.h"
 #include "DNA_space_types.h"
 #include "DNA_windowmanager_types.h" /* for ReportType */
-#include "zlib.h"
 
 struct BLI_mmap_file;
 struct BLOCacheStorage;
@@ -50,7 +50,7 @@ enum eFileDataFlag {
   FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1,
   FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2,
   FD_FLAGS_FILE_OK = 1 << 3,
-  FD_FLAGS_NOT_MY_BUFFER = 1 << 4,
+  FD_FLAGS_IS_MEMFILE = 1 << 4,
   /* XXX Unused in practice (checked once but never set). */
   FD_FLAGS_NOT_MY_LIBMAP = 1 << 5,
 };
@@ -60,44 +60,18 @@ enum eFileDataFlag {
 #  pragma GCC poison off_t
 #endif
 
-#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
-typedef int64_t off64_t;
-#endif
-
-typedef ssize_t(FileDataReadFn)(struct FileData *filedata,
-                                void *buffer,
-                                size_t size,
-                                bool *r_is_memchunk_identical);
-typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
-
 typedef struct FileData {
   /** Linked list of BHeadN's. */
   ListBase bhead_list;
   enum eFileDataFlag flags;
   bool is_eof;
-  size_t buffersize;
-  off64_t file_offset;
 
-  FileDataReadFn *read;
-  FileDataSeekFn *seek;
+  FileReader *file;
 
-  /** Regular file reading. */
-  int filedes;
-
-  /** Variables needed for reading from memory / stream / memory-mapped files. */
-  const char *buffer;
-  struct BLI_mmap_file *mmap_file;
-  /** Variables needed for reading from memfile (undo). */
-  struct MemFile *memfile;
   /** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
    * to detect unchanged data from memfile. */
   int undo_direction; /* eUndoStepDir */
 
-  /** Variables needed for reading from file. */
-  gzFile gzfiledes;
-  /** Gzip stream for memory decompression. */
-  z_stream strm;
-
   /** Now only in use for library appending. */
   char relabase[FILE_MAX];
 

source/blender/blenloader/intern/undofile.c

@@ -48,6 +48,7 @@
 
 #include "BKE_lib_id.h"
 #include "BKE_main.h"
+#include "BKE_undo_system.h"
 
 /* keep last */
 #include "BLI_strict_flags.h"
@@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename)
   }
   return true;
 }
+
+static ssize_t undo_read(FileReader *reader, void *buffer, size_t size)
+{
+  UndoReader *undo = (UndoReader *)reader;
+
+  static size_t seek = SIZE_MAX; /* The current position. */
+  static size_t offset = 0;      /* Size of previous chunks. */
+  static MemFileChunk *chunk = NULL;
+  size_t chunkoffset, readsize, totread;
+
+  undo->memchunk_identical = true;
+
+  if (size == 0) {
+    return 0;
+  }
+
+  if (seek != (size_t)undo->reader.offset) {
+    chunk = undo->memfile->chunks.first;
+    seek = 0;
+
+    while (chunk) {
+      if (seek + chunk->size > (size_t)undo->reader.offset) {
+        break;
+      }
+      seek += chunk->size;
+      chunk = chunk->next;
+    }
+    offset = seek;
+    seek = (size_t)undo->reader.offset;
+  }
+
+  if (chunk) {
+    totread = 0;
+
+    do {
+      /* First check if it's on the end if current chunk. */
+      if (seek - offset == chunk->size) {
+        offset += chunk->size;
+        chunk = chunk->next;
+      }
+
+      /* Debug, should never happen. */
+      if (chunk == NULL) {
+        printf("illegal read, chunk zero\n");
+        return 0;
+      }
+
+      chunkoffset = seek - offset;
+      readsize = size - totread;
+
+      /* Data can be spread over multiple chunks, so clamp size
+       * to within this chunk, and then it will read further in
+       * the next chunk. */
+      if (chunkoffset + readsize > chunk->size) {
+        readsize = chunk->size - chunkoffset;
+      }
+
+      memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
+      totread += readsize;
+      undo->reader.offset += (off64_t)readsize;
+      seek += readsize;
+
+      /* `is_identical` of current chunk represents whether it changed compared to previous undo
+       * step. this is fine in redo case, but not in undo case, where we need an extra flag
+       * defined when saving the next (future) step after the one we want to restore, as we are
+       * supposed to 'come from' that future undo step, and not the one before current one. */
+      undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical :
+                                                                      chunk->is_identical_future;
+    } while (totread < size);
+
+    return (ssize_t)totread;
+  }
+
+  return 0;
+}
+
+static void undo_close(FileReader *reader)
+{
+  MEM_freeN(reader);
+}
+
+FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction)
+{
+  UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__);
+
+  undo->memfile = memfile;
+  undo->undo_direction = undo_direction;
+
+  undo->reader.read = undo_read;
+  undo->reader.seek = NULL;
+  undo->reader.close = undo_close;
+
+  return (FileReader *)undo;
+}

source/blender/blenloader/intern/versioning_250.c

@@ -23,8 +23,7 @@
 #else
 #  include "BLI_winstuff.h"
 #  include "winsock2.h"
-#  include <io.h>   /* for open close read */
-#  include <zlib.h> /* odd include order-issue */
+#  include <io.h> /* for open close read */
 #endif
 
 /* allow readfile to use deprecated functionality */

source/blender/blenloader/intern/versioning_legacy.c

@@ -28,8 +28,7 @@
 #else
 #  include "BLI_winstuff.h"
 #  include "winsock2.h"
-#  include <io.h>   /* for open close read */
-#  include <zlib.h> /* odd include order-issue */
+#  include <io.h> /* for open close read */
 #endif
 
 /* allow readfile to use deprecated functionality */

source/blender/blenloader/intern/writefile.c

@@ -83,7 +83,6 @@
 #  include "BLI_winstuff.h"
 #  include "winsock2.h"
 #  include <io.h>
-#  include <zlib.h> /* odd include order-issue */
 #else
 #  include <unistd.h> /* FreeBSD, for write() and close(). */
 #endif
@@ -101,7 +100,12 @@
 #include "BLI_bitmap.h"
 #include "BLI_blenlib.h"
 #include "BLI_endian_defines.h"
+#include "BLI_endian_switch.h"
+#include "BLI_link_utils.h"
+#include "BLI_linklist.h"
+#include "BLI_math_base.h"
 #include "BLI_mempool.h"
+#include "BLI_threads.h"
 #include "MEM_guardedalloc.h" /* MEM_freeN */
 
 #include "BKE_blender_version.h"
@@ -129,14 +133,21 @@
 
 #include <errno.h>
 
+#include <zstd.h>
+
 /* Make preferences read-only. */
 #define U (*((const UserDef *)&U))
 
 /* ********* my write, buffered writing with minimum size chunks ************ */
 
 /* Use optimal allocation since blocks of this size are kept in memory for undo. */
-#define MYWRITE_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */
-#define MYWRITE_MAX_CHUNK (MEM_SIZE_OPTIMAL(1 << 15))   /* ~32kb */
+#define MEM_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */
+#define MEM_CHUNK_SIZE (MEM_SIZE_OPTIMAL(1 << 15))  /* ~32kb */
+
+#define ZSTD_BUFFER_SIZE (1 << 21) /* 2mb */
+#define ZSTD_CHUNK_SIZE (1 << 20)  /* 1mb */
+
+#define ZSTD_COMPRESSION_LEVEL 3
 
 /** Use if we want to store how many bytes have been written to the file. */
 // #define USE_WRITE_DATA_LEN
@@ -147,9 +158,16 @@
 
 typedef enum {
   WW_WRAP_NONE = 1,
-  WW_WRAP_ZLIB,
+  WW_WRAP_ZSTD,
 } eWriteWrapType;
 
+typedef struct ZstdFrame {
+  struct ZstdFrame *next, *prev;
+
+  uint32_t compressed_size;
+  uint32_t uncompressed_size;
+} ZstdFrame;
+
 typedef struct WriteWrap WriteWrap;
 struct WriteWrap {
   /* callbacks */
@@ -161,15 +179,23 @@ struct WriteWrap {
   bool use_buf;
 
   /* internal */
-  union {
-    int file_handle;
-    gzFile gz_handle;
-  } _user_data;
+  int file_handle;
+  struct {
+    ListBase threadpool;
+    ListBase tasks;
+    ThreadMutex mutex;
+    ThreadCondition condition;
+    int next_frame;
+    int num_frames;
+
+    int level;
+    ListBase frames;
+
+    bool write_error;
+  } zstd;
 };
 
 /* none */
-#define FILE_HANDLE(ww) (ww)->_user_data.file_handle
-
 static bool ww_open_none(WriteWrap *ww, const char *filepath)
 {
   int file;
@@ -177,7 +203,7 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath)
   file = BLI_open(filepath, O_BINARY + O_WRONLY + O_CREAT + O_TRUNC, 0666);
 
   if (file != -1) {
-    FILE_HANDLE(ww) = file;
+    ww->file_handle = file;
     return true;
   }
 
@@ -185,39 +211,170 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath)
 }
 static bool ww_close_none(WriteWrap *ww)
 {
-  return (close(FILE_HANDLE(ww)) != -1);
+  return (close(ww->file_handle) != -1);
 }
 static size_t ww_write_none(WriteWrap *ww, const char *buf, size_t buf_len)
 {
-  return write(FILE_HANDLE(ww), buf, buf_len);
+  return write(ww->file_handle, buf, buf_len);
 }
-#undef FILE_HANDLE
 
-/* zlib */
-#define FILE_HANDLE(ww) (ww)->_user_data.gz_handle
+/* zstd */
 
-static bool ww_open_zlib(WriteWrap *ww, const char *filepath)
+typedef struct {
+  struct ZstdWriteBlockTask *next, *prev;
+  void *data;
+  size_t size;
+  int frame_number;
+  WriteWrap *ww;
+} ZstdWriteBlockTask;
+
+static void *zstd_write_task(void *userdata)
 {
-  gzFile file;
+  ZstdWriteBlockTask *task = userdata;
+  WriteWrap *ww = task->ww;
 
-  file = BLI_gzopen(filepath, "wb1");
+  size_t out_buf_len = ZSTD_compressBound(task->size);
+  void *out_buf = MEM_mallocN(out_buf_len, "Zstd out buffer");
+  size_t out_size = ZSTD_compress(
+      out_buf, out_buf_len, task->data, task->size, ZSTD_COMPRESSION_LEVEL);
 
-  if (file != Z_NULL) {
-    FILE_HANDLE(ww) = file;
-    return true;
+  MEM_freeN(task->data);
+
+  BLI_mutex_lock(&ww->zstd.mutex);
+
+  while (ww->zstd.next_frame != task->frame_number) {
+    BLI_condition_wait(&ww->zstd.condition, &ww->zstd.mutex);
   }
 
-  return false;
+  if (ZSTD_isError(out_size)) {
+    ww->zstd.write_error = true;
+  }
+  else {
+    if (ww_write_none(ww, out_buf, out_size) == out_size) {
+      ZstdFrame *frameinfo = MEM_mallocN(sizeof(ZstdFrame), "zstd frameinfo");
+      frameinfo->uncompressed_size = task->size;
+      frameinfo->compressed_size = out_size;
+      BLI_addtail(&ww->zstd.frames, frameinfo);
+    }
+    else {
+      ww->zstd.write_error = true;
+    }
+  }
+
+  ww->zstd.next_frame++;
+
+  BLI_mutex_unlock(&ww->zstd.mutex);
+  BLI_condition_notify_all(&ww->zstd.condition);
+
+  MEM_freeN(out_buf);
+  return NULL;
 }
-static bool ww_close_zlib(WriteWrap *ww)
+
+static bool ww_open_zstd(WriteWrap *ww, const char *filepath)
 {
-  return (gzclose(FILE_HANDLE(ww)) == Z_OK);
+  if (!ww_open_none(ww, filepath)) {
+    return false;
+  }
+
+  /* Leave one thread open for the main writing logic, unless we only have one HW thread. */
+  int num_threads = max_ii(1, BLI_system_thread_count() - 1);
+  BLI_threadpool_init(&ww->zstd.threadpool, zstd_write_task, num_threads);
+  BLI_mutex_init(&ww->zstd.mutex);
+  BLI_condition_init(&ww->zstd.condition);
+
+  return true;
 }
-static size_t ww_write_zlib(WriteWrap *ww, const char *buf, size_t buf_len)
+
+static void zstd_write_u32_le(WriteWrap *ww, uint32_t val)
 {
-  return gzwrite(FILE_HANDLE(ww), buf, buf_len);
+#ifdef __BIG_ENDIAN__
+  BLI_endian_switch_uint32(&val);
+#endif
+  ww_write_none(ww, (char *)&val, sizeof(uint32_t));
+}
+
+/* In order to implement efficient seeking when reading the .blend, we append
+ * a skippable frame that encodes information about the other frames present
+ * in the file.
+ * The format here follows the upstream spec for seekable files:
+ * https://github.com/facebook/zstd/blob/master/contrib/seekable_format/zstd_seekable_compression_format.md
+ * If this information is not present in a file (e.g. if it was compressed
+ * with external tools), it can still be opened in Blender, but seeking will
+ * not be supported, so more memory might be needed. */
+static void zstd_write_seekable_frames(WriteWrap *ww)
+{
+  /* Write seek table header (magic number and frame size). */
+  zstd_write_u32_le(ww, 0x184D2A5E);
+
+  /* The actual frame number might not match ww->zstd.num_frames if there was a write error. */
+  const uint32_t num_frames = BLI_listbase_count(&ww->zstd.frames);
+  /* Each frame consists of two u32, so 8 bytes each.
+   * After the frames, a footer containing two u32 and one byte (9 bytes total) is written. */
+  const uint32_t frame_size = num_frames * 8 + 9;
+  zstd_write_u32_le(ww, frame_size);
+
+  /* Write seek table entries. */
+  LISTBASE_FOREACH (ZstdFrame *, frame, &ww->zstd.frames) {
+    zstd_write_u32_le(ww, frame->compressed_size);
+    zstd_write_u32_le(ww, frame->uncompressed_size);
+  }
+
+  /* Write seek table footer (number of frames, option flags and second magic number). */
+  zstd_write_u32_le(ww, num_frames);
+  const char flags = 0; /* We don't store checksums for each frame. */
+  ww_write_none(ww, &flags, 1);
+  zstd_write_u32_le(ww, 0x8F92EAB1);
+}
+
+static bool ww_close_zstd(WriteWrap *ww)
+{
+  BLI_threadpool_end(&ww->zstd.threadpool);
+  BLI_freelistN(&ww->zstd.tasks);
+
+  BLI_mutex_end(&ww->zstd.mutex);
+  BLI_condition_end(&ww->zstd.condition);
+
+  zstd_write_seekable_frames(ww);
+  BLI_freelistN(&ww->zstd.frames);
+
+  return ww_close_none(ww) && !ww->zstd.write_error;
+}
+
+static size_t ww_write_zstd(WriteWrap *ww, const char *buf, size_t buf_len)
+{
+  if (ww->zstd.write_error) {
+    return 0;
+  }
+
+  ZstdWriteBlockTask *task = MEM_mallocN(sizeof(ZstdWriteBlockTask), __func__);
+  task->data = MEM_mallocN(buf_len, __func__);
+  memcpy(task->data, buf, buf_len);
+  task->size = buf_len;
+  task->frame_number = ww->zstd.num_frames++;
+  task->ww = ww;
+
+  BLI_mutex_lock(&ww->zstd.mutex);
+  BLI_addtail(&ww->zstd.tasks, task);
+
+  /* If there's a free worker thread, just push the block into that thread.
+   * Otherwise, we wait for the earliest thread to finish.
+   * We look up the earliest thread while holding the mutex, but release it
+   * before joining the thread to prevent a deadlock. */
+  ZstdWriteBlockTask *first_task = ww->zstd.tasks.first;
+  BLI_mutex_unlock(&ww->zstd.mutex);
+  if (!BLI_available_threads(&ww->zstd.threadpool)) {
+    BLI_threadpool_remove(&ww->zstd.threadpool, first_task);
+
+    /* If the task list was empty before we pushed our task, there should
+     * always be a free thread. */
+    BLI_assert(first_task != task);
+    BLI_remlink(&ww->zstd.tasks, first_task);
+    MEM_freeN(first_task);
+  }
+  BLI_threadpool_insert(&ww->zstd.threadpool, task);
+
+  return buf_len;
 }
-#undef FILE_HANDLE
 
 /* --- end compression types --- */
 
@@ -226,11 +383,11 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww)
   memset(r_ww, 0, sizeof(*r_ww));
 
   switch (ww_type) {
-    case WW_WRAP_ZLIB: {
-      r_ww->open = ww_open_zlib;
-      r_ww->close = ww_close_zlib;
-      r_ww->write = ww_write_zlib;
-      r_ww->use_buf = false;
+    case WW_WRAP_ZSTD: {
+      r_ww->open = ww_open_zstd;
+      r_ww->close = ww_close_zstd;
+      r_ww->write = ww_write_zstd;
+      r_ww->use_buf = true;
       break;
     }
     default: {
@@ -252,10 +409,17 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww)
 typedef struct {
   const struct SDNA *sdna;
 
-  /** Use for file and memory writing (fixed size of #MYWRITE_BUFFER_SIZE). */
-  uchar *buf;
-  /** Number of bytes used in #WriteData.buf (flushed when exceeded). */
-  size_t buf_used_len;
+  struct {
+    /** Use for file and memory writing (size stored in max_size). */
+    uchar *buf;
+    /** Number of bytes used in #WriteData.buf (flushed when exceeded). */
+    size_t used_len;
+
+    /** Maximum size of the buffer. */
+    size_t max_size;
+    /** Threshold above which writes get their own chunk. */
+    size_t chunk_size;
+  } buffer;
 
 #ifdef USE_WRITE_DATA_LEN
   /** Total number of bytes written. */
@@ -271,7 +435,7 @@ typedef struct {
   bool use_memfile;
 
   /**
-   * Wrap writing, so we can use zlib or
+   * Wrap writing, so we can use zstd or
    * other compression types later, see: G_FILE_COMPRESS
    * Will be NULL for UNDO.
    */
@@ -291,7 +455,15 @@ static WriteData *writedata_new(WriteWrap *ww)
   wd->ww = ww;
 
   if ((ww == NULL) || (ww->use_buf)) {
-    wd->buf = MEM_mallocN(MYWRITE_BUFFER_SIZE, "wd->buf");
+    if (ww == NULL) {
+      wd->buffer.max_size = MEM_BUFFER_SIZE;
+      wd->buffer.chunk_size = MEM_CHUNK_SIZE;
+    }
+    else {
+      wd->buffer.max_size = ZSTD_BUFFER_SIZE;
+      wd->buffer.chunk_size = ZSTD_CHUNK_SIZE;
+    }
+    wd->buffer.buf = MEM_mallocN(wd->buffer.max_size, "wd->buffer.buf");
   }
 
   return wd;
@@ -325,8 +497,8 @@ static void writedata_do_write(WriteData *wd, const void *mem, size_t memlen)
 
 static void writedata_free(WriteData *wd)
 {
-  if (wd->buf) {
-    MEM_freeN(wd->buf);
+  if (wd->buffer.buf) {
+    MEM_freeN(wd->buffer.buf);
   }
   MEM_freeN(wd);
 }
@@ -343,9 +515,9 @@ static void writedata_free(WriteData *wd)
  */
 static void mywrite_flush(WriteData *wd)
 {
-  if (wd->buf_used_len != 0) {
-    writedata_do_write(wd, wd->buf, wd->buf_used_len);
-    wd->buf_used_len = 0;
+  if (wd->buffer.used_len != 0) {
+    writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+    wd->buffer.used_len = 0;
   }
 }
 
@@ -369,20 +541,20 @@ static void mywrite(WriteData *wd, const void *adr, size_t len)
   wd->write_len += len;
 #endif
 
-  if (wd->buf == NULL) {
+  if (wd->buffer.buf == NULL) {
     writedata_do_write(wd, adr, len);
   }
   else {
     /* if we have a single big chunk, write existing data in
      * buffer and write out big chunk in smaller pieces */
-    if (len > MYWRITE_MAX_CHUNK) {
-      if (wd->buf_used_len != 0) {
-        writedata_do_write(wd, wd->buf, wd->buf_used_len);
-        wd->buf_used_len = 0;
+    if (len > wd->buffer.chunk_size) {
+      if (wd->buffer.used_len != 0) {
+        writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+        wd->buffer.used_len = 0;
       }
 
       do {
-        size_t writelen = MIN2(len, MYWRITE_MAX_CHUNK);
+        size_t writelen = MIN2(len, wd->buffer.chunk_size);
         writedata_do_write(wd, adr, writelen);
         adr = (const char *)adr + writelen;
         len -= writelen;
@@ -392,14 +564,14 @@ static void mywrite(WriteData *wd, const void *adr, size_t len)
     }
 
     /* if data would overflow buffer, write out the buffer */
-    if (len + wd->buf_used_len > MYWRITE_BUFFER_SIZE - 1) {
-      writedata_do_write(wd, wd->buf, wd->buf_used_len);
-      wd->buf_used_len = 0;
+    if (len + wd->buffer.used_len > wd->buffer.max_size - 1) {
+      writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+      wd->buffer.used_len = 0;
     }
 
     /* append data at end of buffer */
-    memcpy(&wd->buf[wd->buf_used_len], adr, len);
-    wd->buf_used_len += len;
+    memcpy(&wd->buffer.buf[wd->buffer.used_len], adr, len);
+    wd->buffer.used_len += len;
   }
 }
 
@@ -430,9 +602,9 @@ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *curren
  */
 static bool mywrite_end(WriteData *wd)
 {
-  if (wd->buf_used_len != 0) {
-    writedata_do_write(wd, wd->buf, wd->buf_used_len);
-    wd->buf_used_len = 0;
+  if (wd->buffer.used_len != 0) {
+    writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+    wd->buffer.used_len = 0;
   }
 
   if (wd->use_memfile) {
@@ -1150,7 +1322,6 @@ bool BLO_write_file(Main *mainvar,
                     ReportList *reports)
 {
   char tempname[FILE_MAX + 1];
-  eWriteWrapType ww_type;
   WriteWrap ww;
 
   eBLO_WritePathRemap remap_mode = params->remap_mode;
@@ -1172,14 +1343,7 @@ bool BLO_write_file(Main *mainvar,
   /* open temporary file, so we preserve the original in case we crash */
   BLI_snprintf(tempname, sizeof(tempname), "%s@", filepath);
 
-  if (write_flags & G_FILE_COMPRESS) {
-    ww_type = WW_WRAP_ZLIB;
-  }
-  else {
-    ww_type = WW_WRAP_NONE;
-  }
-
-  ww_handle_init(ww_type, &ww);
+  ww_handle_init((write_flags & G_FILE_COMPRESS) ? WW_WRAP_ZSTD : WW_WRAP_NONE, &ww);
 
   if (ww.open(&ww, tempname) == false) {
     BKE_reportf(

source/blender/compositor/CMakeLists.txt

@@ -328,10 +328,14 @@ set(SRC
   operations/COM_FastGaussianBlurOperation.h
   operations/COM_GammaCorrectOperation.cc
   operations/COM_GammaCorrectOperation.h
+  operations/COM_GaussianAlphaBlurBaseOperation.cc
+  operations/COM_GaussianAlphaBlurBaseOperation.h
   operations/COM_GaussianAlphaXBlurOperation.cc
   operations/COM_GaussianAlphaXBlurOperation.h
   operations/COM_GaussianAlphaYBlurOperation.cc
   operations/COM_GaussianAlphaYBlurOperation.h
+  operations/COM_GaussianBlurBaseOperation.cc
+  operations/COM_GaussianBlurBaseOperation.h
   operations/COM_GaussianBokehBlurOperation.cc
   operations/COM_GaussianBokehBlurOperation.h
   operations/COM_GaussianXBlurOperation.cc
@@ -515,6 +519,8 @@ set(SRC
   operations/COM_ScaleOperation.h
   operations/COM_ScreenLensDistortionOperation.cc
   operations/COM_ScreenLensDistortionOperation.h
+  operations/COM_TransformOperation.cc
+  operations/COM_TransformOperation.h
   operations/COM_TranslateOperation.cc
   operations/COM_TranslateOperation.h
   operations/COM_WrapOperation.cc

source/blender/compositor/COM_defines.h

@@ -33,6 +33,8 @@ enum class eExecutionModel {
   FullFrame
 };
 
+enum class eDimension { X, Y };
+
 /**
  * \brief possible data types for sockets
  * \ingroup Model
@@ -119,6 +121,8 @@ constexpr float COM_PREVIEW_SIZE = 140.f;
 constexpr float COM_RULE_OF_THIRDS_DIVIDER = 100.0f;
 constexpr float COM_BLUR_BOKEH_PIXELS = 512;
 
+constexpr rcti COM_SINGLE_ELEM_AREA = {0, 1, 0, 1};
+
 constexpr IndexRange XRange(const rcti &area)
 {
   return IndexRange(area.xmin, area.xmax - area.xmin);

source/blender/compositor/intern/COM_CompositorContext.h

@@ -239,6 +239,12 @@ class CompositorContext {
     this->m_hasActiveOpenCLDevices = hasAvtiveOpenCLDevices;
   }
 
+  /** Whether it has a view with a specific name and not the default one. */
+  bool has_explicit_view() const
+  {
+    return m_viewName && m_viewName[0] != '\0';
+  }
+
   /**
    * \brief get the active rendering view
    */

source/blender/compositor/intern/COM_MemoryBuffer.cc

@@ -245,7 +245,9 @@ void MemoryBuffer::copy_from(const MemoryBuffer *src,
 
 void MemoryBuffer::copy_from(const uchar *src, const rcti &area)
 {
-  copy_from(src, area, 0, this->get_num_channels(), this->get_num_channels(), 0);
+  const int elem_stride = this->get_num_channels();
+  const int row_stride = elem_stride * getWidth();
+  copy_from(src, area, 0, this->get_num_channels(), elem_stride, row_stride, 0);
 }
 
 void MemoryBuffer::copy_from(const uchar *src,
@@ -253,10 +255,18 @@ void MemoryBuffer::copy_from(const uchar *src,
                              const int channel_offset,
                              const int elem_size,
                              const int elem_stride,
+                             const int row_stride,
                              const int to_channel_offset)
 {
-  copy_from(
-      src, area, channel_offset, elem_size, elem_stride, area.xmin, area.ymin, to_channel_offset);
+  copy_from(src,
+            area,
+            channel_offset,
+            elem_size,
+            elem_stride,
+            row_stride,
+            area.xmin,
+            area.ymin,
+            to_channel_offset);
 }
 
 void MemoryBuffer::copy_from(const uchar *src,
@@ -264,6 +274,7 @@ void MemoryBuffer::copy_from(const uchar *src,
                              const int channel_offset,
                              const int elem_size,
                              const int elem_stride,
+                             const int row_stride,
                              const int to_x,
                              const int to_y,
                              const int to_channel_offset)
@@ -273,10 +284,9 @@ void MemoryBuffer::copy_from(const uchar *src,
 
   const int width = BLI_rcti_size_x(&area);
   const int height = BLI_rcti_size_y(&area);
-  const int src_row_stride = width * elem_stride;
-  const uchar *const src_start = src + area.ymin * src_row_stride + channel_offset;
+  const uchar *const src_start = src + area.ymin * row_stride + channel_offset;
   for (int y = 0; y < height; y++) {
-    const uchar *from_elem = src_start + y * src_row_stride;
+    const uchar *from_elem = src_start + y * row_stride + area.xmin * elem_stride;
     float *to_elem = &this->get_value(to_x, to_y + y, to_channel_offset);
     const float *row_end = to_elem + width * this->elem_stride;
     while (to_elem < row_end) {
@@ -346,7 +356,16 @@ void MemoryBuffer::copy_from(const ImBuf *src,
   else if (src->rect) {
     const uchar *uc_buf = (uchar *)src->rect;
     const int elem_stride = src->channels;
-    copy_from(uc_buf, area, channel_offset, elem_size, elem_stride, to_x, to_y, to_channel_offset);
+    const int row_stride = elem_stride * src->x;
+    copy_from(uc_buf,
+              area,
+              channel_offset,
+              elem_size,
+              elem_stride,
+              row_stride,
+              to_x,
+              to_y,
+              to_channel_offset);
     if (ensure_linear_space) {
       colorspace_to_scene_linear(this, area, src->rect_colorspace);
     }
@@ -405,12 +424,48 @@ void MemoryBuffer::addPixel(int x, int y, const float color[4])
   }
 }
 
+static void read_ewa_elem(void *userdata, int x, int y, float result[4])
+{
+  const MemoryBuffer *buffer = static_cast<const MemoryBuffer *>(userdata);
+  buffer->read_elem_checked(x, y, result);
+}
+
+void MemoryBuffer::read_elem_filtered(
+    const float x, const float y, float dx[2], float dy[2], float *out) const
+{
+  BLI_assert(this->m_datatype == DataType::Color);
+
+  const float deriv[2][2] = {{dx[0], dx[1]}, {dy[0], dy[1]}};
+
+  float inv_width = 1.0f / (float)this->getWidth(), inv_height = 1.0f / (float)this->getHeight();
+  /* TODO(sergey): Render pipeline uses normalized coordinates and derivatives,
+   * but compositor uses pixel space. For now let's just divide the values and
+   * switch compositor to normalized space for EWA later.
+   */
+  float uv_normal[2] = {get_relative_x(x) * inv_width, get_relative_y(y) * inv_height};
+  float du_normal[2] = {deriv[0][0] * inv_width, deriv[0][1] * inv_height};
+  float dv_normal[2] = {deriv[1][0] * inv_width, deriv[1][1] * inv_height};
+
+  BLI_ewa_filter(this->getWidth(),
+                 this->getHeight(),
+                 false,
+                 true,
+                 uv_normal,
+                 du_normal,
+                 dv_normal,
+                 read_ewa_elem,
+                 const_cast<MemoryBuffer *>(this),
+                 out);
+}
+
+/* TODO(manzanilla): to be removed with tiled implementation. */
 static void read_ewa_pixel_sampled(void *userdata, int x, int y, float result[4])
 {
   MemoryBuffer *buffer = (MemoryBuffer *)userdata;
   buffer->read(result, x, y);
 }
 
+/* TODO(manzanilla): to be removed with tiled implementation. */
 void MemoryBuffer::readEWA(float *result, const float uv[2], const float derivatives[2][2])
 {
   if (m_is_a_single_elem) {

source/blender/compositor/intern/COM_MemoryBuffer.h

@@ -191,23 +191,96 @@ class MemoryBuffer {
 
   void read_elem(int x, int y, float *out) const
   {
-    memcpy(out, get_elem(x, y), m_num_channels * sizeof(float));
+    memcpy(out, get_elem(x, y), get_elem_bytes_len());
+  }
+
+  void read_elem_checked(int x, int y, float *out) const
+  {
+    if (x < m_rect.xmin || x >= m_rect.xmax || y < m_rect.ymin || y >= m_rect.ymax) {
+      clear_elem(out);
+    }
+    else {
+      read_elem(x, y, out);
+    }
+  }
+
+  void read_elem_checked(float x, float y, float *out) const
+  {
+    if (x < m_rect.xmin || x >= m_rect.xmax || y < m_rect.ymin || y >= m_rect.ymax) {
+      clear_elem(out);
+    }
+    else {
+      read_elem(x, y, out);
+    }
+  }
+
+  void read_elem_bilinear(float x, float y, float *out) const
+  {
+    /* Only clear past +/-1 borders to be able to smooth edges. */
+    if (x <= m_rect.xmin - 1.0f || x >= m_rect.xmax || y <= m_rect.ymin - 1.0f ||
+        y >= m_rect.ymax) {
+      clear_elem(out);
+      return;
+    }
+
+    if (m_is_a_single_elem) {
+      if (x >= m_rect.xmin && x < m_rect.xmax - 1.0f && y >= m_rect.ymin &&
+          y < m_rect.ymax - 1.0f) {
+        memcpy(out, m_buffer, get_elem_bytes_len());
+        return;
+      }
+
+      /* Do sampling at borders to smooth edges. */
+      const float last_x = getWidth() - 1.0f;
+      const float rel_x = get_relative_x(x);
+      float single_x = 0.0f;
+      if (rel_x < 0.0f) {
+        single_x = rel_x;
+      }
+      else if (rel_x > last_x) {
+        single_x = rel_x - last_x;
+      }
+
+      const float last_y = getHeight() - 1.0f;
+      const float rel_y = get_relative_y(y);
+      float single_y = 0.0f;
+      if (rel_y < 0.0f) {
+        single_y = rel_y;
+      }
+      else if (rel_y > last_y) {
+        single_y = rel_y - last_y;
+      }
+
+      BLI_bilinear_interpolation_fl(m_buffer, out, 1, 1, m_num_channels, single_x, single_y);
+      return;
+    }
+
+    BLI_bilinear_interpolation_fl(m_buffer,
+                                  out,
+                                  getWidth(),
+                                  getHeight(),
+                                  m_num_channels,
+                                  get_relative_x(x),
+                                  get_relative_y(y));
   }
 
   void read_elem_sampled(float x, float y, PixelSampler sampler, float *out) const
   {
     switch (sampler) {
       case PixelSampler::Nearest:
-        this->read_elem(x, y, out);
+        read_elem_checked(x, y, out);
         break;
       case PixelSampler::Bilinear:
       case PixelSampler::Bicubic:
         /* No bicubic. Current implementation produces fuzzy results. */
-        this->readBilinear(out, x, y);
+        read_elem_bilinear(x, y, out);
         break;
     }
   }
 
+  void read_elem_filtered(
+      const float x, const float y, float dx[2], float dy[2], float *out) const;
+
   /**
    * Get channel value at given coordinates.
    */
@@ -403,6 +476,8 @@ class MemoryBuffer {
     y = y + m_rect.ymin;
   }
 
+  /* TODO(manzanilla): to be removed with tiled implementation. For applying #MemoryBufferExtend
+   * use #wrap_pixel. */
   inline void read(float *result,
                    int x,
                    int y,
@@ -425,6 +500,7 @@ class MemoryBuffer {
     }
   }
 
+  /* TODO(manzanilla): to be removed with tiled implementation. */
   inline void readNoCheck(float *result,
                           int x,
                           int y,
@@ -507,12 +583,14 @@ class MemoryBuffer {
                  int channel_offset,
                  int elem_size,
                  int elem_stride,
+                 int row_stride,
                  int to_channel_offset);
   void copy_from(const uchar *src,
                  const rcti &area,
                  int channel_offset,
                  int elem_size,
                  int elem_stride,
+                 int row_stride,
                  int to_x,
                  int to_y,
                  int to_channel_offset);
@@ -582,6 +660,21 @@ class MemoryBuffer {
     return get_memory_width() * get_memory_height();
   }
 
+  void clear_elem(float *out) const
+  {
+    memset(out, 0, this->m_num_channels * sizeof(float));
+  }
+
+  template<typename T> T get_relative_x(T x) const
+  {
+    return x - m_rect.xmin;
+  }
+
+  template<typename T> T get_relative_y(T y) const
+  {
+    return y - m_rect.ymin;
+  }
+
   void copy_single_elem_from(const MemoryBuffer *src,
                              int channel_offset,
                              int elem_size,

source/blender/compositor/intern/COM_NodeGraph.cc

@@ -248,7 +248,9 @@ void NodeGraph::add_proxies_group_inputs(bNode *b_node, bNode *b_node_io)
   }
 }
 
-void NodeGraph::add_proxies_group_outputs(bNode *b_node, bNode *b_node_io, bool use_buffer)
+void NodeGraph::add_proxies_group_outputs(const CompositorContext &context,
+                                          bNode *b_node,
+                                          bNode *b_node_io)
 {
   bNodeTree *b_group_tree = (bNodeTree *)b_node->id;
   BLI_assert(b_group_tree); /* should have been checked in advance */
@@ -261,7 +263,8 @@ void NodeGraph::add_proxies_group_outputs(bNode *b_node, bNode *b_node_io, bool
        b_sock_io = b_sock_io->next) {
     bNodeSocket *b_sock_group = find_b_node_output(b_node, b_sock_io->identifier);
     if (b_sock_group) {
-      if (use_buffer) {
+      if (context.isGroupnodeBufferEnabled() &&
+          context.get_execution_model() == eExecutionModel::Tiled) {
         SocketBufferNode *buffer = new SocketBufferNode(b_node_io, b_sock_io, b_sock_group);
         add_node(buffer, b_group_tree, key, is_active_group);
       }
@@ -297,7 +300,7 @@ void NodeGraph::add_proxies_group(const CompositorContext &context,
     }
 
     if (b_node_io->type == NODE_GROUP_OUTPUT && (b_node_io->flag & NODE_DO_OUTPUT)) {
-      add_proxies_group_outputs(b_node, b_node_io, context.isGroupnodeBufferEnabled());
+      add_proxies_group_outputs(context, b_node, b_node_io);
     }
   }
 

source/blender/compositor/intern/COM_NodeGraph.h

@@ -107,7 +107,9 @@ class NodeGraph {
                         bool is_active_group);
 
   void add_proxies_group_inputs(bNode *b_node, bNode *b_node_io);
-  void add_proxies_group_outputs(bNode *b_node, bNode *b_node_io, bool use_buffer);
+  void add_proxies_group_outputs(const CompositorContext &context,
+                                 bNode *b_node,
+                                 bNode *b_node_io);
   void add_proxies_group(const CompositorContext &context, bNode *b_node, bNodeInstanceKey key);
 
   void add_proxies_reroute(bNodeTree *b_ntree,

source/blender/compositor/intern/COM_NodeOperation.cc

@@ -82,8 +82,12 @@ void NodeOperation::determineResolution(unsigned int resolution[2],
     input.determineResolution(resolution, preferredResolution);
     used_resolution_index = m_resolutionInputSocketIndex;
   }
-  unsigned int temp2[2] = {resolution[0], resolution[1]};
 
+  if (modify_determined_resolution_fn_) {
+    modify_determined_resolution_fn_(resolution);
+  }
+
+  unsigned int temp2[2] = {resolution[0], resolution[1]};
   unsigned int temp[2];
   for (unsigned int index = 0; index < m_inputs.size(); index++) {
     if (index == used_resolution_index) {

source/blender/compositor/intern/COM_NodeOperation.h

@@ -287,6 +287,8 @@ class NodeOperation {
    */
   unsigned int m_resolutionInputSocketIndex;
 
+  std::function<void(unsigned int resolution[2])> modify_determined_resolution_fn_;
+
   /**
    * \brief mutex reference for very special node initializations
    * \note only use when you really know what you are doing.
@@ -517,6 +519,15 @@ class NodeOperation {
    */
   void setResolutionInputSocketIndex(unsigned int index);
 
+  /**
+   * Set a custom function to modify determined resolution from main input just before setting it
+   * as preferred resolution for the other inputs.
+   */
+  void set_determined_resolution_modifier(std::function<void(unsigned int resolution[2])> fn)
+  {
+    modify_determined_resolution_fn_ = fn;
+  }
+
   /**
    * \brief get the render priority of this node.
    * \note only applicable for output operations like ViewerOperation

source/blender/compositor/nodes/COM_CryptomatteNode.cc

@@ -124,6 +124,10 @@ void CryptomatteNode::input_operations_from_render_source(
     RenderLayer *render_layer = RE_GetRenderLayer(render_result, view_layer->name);
     if (render_layer) {
       LISTBASE_FOREACH (RenderPass *, render_pass, &render_layer->passes) {
+        if (context.has_explicit_view() && !STREQ(render_pass->view, context.getViewName())) {
+          continue;
+        }
+
         const std::string combined_name = combined_layer_pass_name(render_layer, render_pass);
         if (blender::StringRef(combined_name).startswith(prefix)) {
           RenderLayersProg *op = new RenderLayersProg(

source/blender/compositor/nodes/COM_RotateNode.cc

@@ -30,20 +30,31 @@ RotateNode::RotateNode(bNode *editorNode) : Node(editorNode)
 }
 
 void RotateNode::convertToOperations(NodeConverter &converter,
-                                     const CompositorContext & /*context*/) const
+                                     const CompositorContext &context) const
 {
   NodeInput *inputSocket = this->getInputSocket(0);
   NodeInput *inputDegreeSocket = this->getInputSocket(1);
   NodeOutput *outputSocket = this->getOutputSocket(0);
   RotateOperation *operation = new RotateOperation();
-  SetSamplerOperation *sampler = new SetSamplerOperation();
-  sampler->setSampler((PixelSampler)this->getbNode()->custom1);
-
-  converter.addOperation(sampler);
   converter.addOperation(operation);
 
-  converter.addLink(sampler->getOutputSocket(), operation->getInputSocket(0));
-  converter.mapInputSocket(inputSocket, sampler->getInputSocket(0));
+  PixelSampler sampler = (PixelSampler)this->getbNode()->custom1;
+  switch (context.get_execution_model()) {
+    case eExecutionModel::Tiled: {
+      SetSamplerOperation *sampler_op = new SetSamplerOperation();
+      sampler_op->setSampler(sampler);
+      converter.addOperation(sampler_op);
+      converter.addLink(sampler_op->getOutputSocket(), operation->getInputSocket(0));
+      converter.mapInputSocket(inputSocket, sampler_op->getInputSocket(0));
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      operation->set_sampler(sampler);
+      converter.mapInputSocket(inputSocket, operation->getInputSocket(0));
+      break;
+    }
+  }
+
   converter.mapInputSocket(inputDegreeSocket, operation->getInputSocket(1));
   converter.mapOutputSocket(outputSocket, operation->getOutputSocket(0));
 }

source/blender/compositor/nodes/COM_Stabilize2dNode.cc

@@ -22,6 +22,7 @@
 #include "COM_RotateOperation.h"
 #include "COM_ScaleOperation.h"
 #include "COM_SetSamplerOperation.h"
+#include "COM_TransformOperation.h"
 #include "COM_TranslateOperation.h"
 
 #include "BKE_tracking.h"
@@ -42,18 +43,12 @@ void Stabilize2dNode::convertToOperations(NodeConverter &converter,
   NodeInput *imageInput = this->getInputSocket(0);
   MovieClip *clip = (MovieClip *)editorNode->id;
   bool invert = (editorNode->custom2 & CMP_NODEFLAG_STABILIZE_INVERSE) != 0;
+  const PixelSampler sampler = (PixelSampler)editorNode->custom1;
 
-  ScaleRelativeOperation *scaleOperation = new ScaleRelativeOperation();
-  scaleOperation->setSampler((PixelSampler)editorNode->custom1);
-  RotateOperation *rotateOperation = new RotateOperation();
-  rotateOperation->setDoDegree2RadConversion(false);
-  TranslateOperation *translateOperation = new TranslateOperation();
   MovieClipAttributeOperation *scaleAttribute = new MovieClipAttributeOperation();
   MovieClipAttributeOperation *angleAttribute = new MovieClipAttributeOperation();
   MovieClipAttributeOperation *xAttribute = new MovieClipAttributeOperation();
   MovieClipAttributeOperation *yAttribute = new MovieClipAttributeOperation();
-  SetSamplerOperation *psoperation = new SetSamplerOperation();
-  psoperation->setSampler((PixelSampler)editorNode->custom1);
 
   scaleAttribute->setAttribute(MCA_SCALE);
   scaleAttribute->setFramenumber(context.getFramenumber());
@@ -79,38 +74,67 @@ void Stabilize2dNode::convertToOperations(NodeConverter &converter,
   converter.addOperation(angleAttribute);
   converter.addOperation(xAttribute);
   converter.addOperation(yAttribute);
-  converter.addOperation(scaleOperation);
-  converter.addOperation(translateOperation);
-  converter.addOperation(rotateOperation);
-  converter.addOperation(psoperation);
 
-  converter.addLink(scaleAttribute->getOutputSocket(), scaleOperation->getInputSocket(1));
-  converter.addLink(scaleAttribute->getOutputSocket(), scaleOperation->getInputSocket(2));
+  switch (context.get_execution_model()) {
+    case eExecutionModel::Tiled: {
+      ScaleRelativeOperation *scaleOperation = new ScaleRelativeOperation();
+      scaleOperation->setSampler(sampler);
+      RotateOperation *rotateOperation = new RotateOperation();
+      rotateOperation->setDoDegree2RadConversion(false);
+      TranslateOperation *translateOperation = new TranslateOperation();
+      SetSamplerOperation *psoperation = new SetSamplerOperation();
+      psoperation->setSampler(sampler);
 
-  converter.addLink(angleAttribute->getOutputSocket(), rotateOperation->getInputSocket(1));
+      converter.addOperation(scaleOperation);
+      converter.addOperation(translateOperation);
+      converter.addOperation(rotateOperation);
+      converter.addOperation(psoperation);
 
-  converter.addLink(xAttribute->getOutputSocket(), translateOperation->getInputSocket(1));
-  converter.addLink(yAttribute->getOutputSocket(), translateOperation->getInputSocket(2));
+      converter.addLink(scaleAttribute->getOutputSocket(), scaleOperation->getInputSocket(1));
+      converter.addLink(scaleAttribute->getOutputSocket(), scaleOperation->getInputSocket(2));
 
-  converter.mapOutputSocket(getOutputSocket(), psoperation->getOutputSocket());
+      converter.addLink(angleAttribute->getOutputSocket(), rotateOperation->getInputSocket(1));
 
-  if (invert) {
-    // Translate -> Rotate -> Scale.
-    converter.mapInputSocket(imageInput, translateOperation->getInputSocket(0));
+      converter.addLink(xAttribute->getOutputSocket(), translateOperation->getInputSocket(1));
+      converter.addLink(yAttribute->getOutputSocket(), translateOperation->getInputSocket(2));
 
-    converter.addLink(translateOperation->getOutputSocket(), rotateOperation->getInputSocket(0));
-    converter.addLink(rotateOperation->getOutputSocket(), scaleOperation->getInputSocket(0));
+      converter.mapOutputSocket(getOutputSocket(), psoperation->getOutputSocket());
 
-    converter.addLink(scaleOperation->getOutputSocket(), psoperation->getInputSocket(0));
-  }
-  else {
-    // Scale  -> Rotate -> Translate.
-    converter.mapInputSocket(imageInput, scaleOperation->getInputSocket(0));
+      if (invert) {
+        // Translate -> Rotate -> Scale.
+        converter.mapInputSocket(imageInput, translateOperation->getInputSocket(0));
 
-    converter.addLink(scaleOperation->getOutputSocket(), rotateOperation->getInputSocket(0));
-    converter.addLink(rotateOperation->getOutputSocket(), translateOperation->getInputSocket(0));
+        converter.addLink(translateOperation->getOutputSocket(),
+                          rotateOperation->getInputSocket(0));
+        converter.addLink(rotateOperation->getOutputSocket(), scaleOperation->getInputSocket(0));
 
-    converter.addLink(translateOperation->getOutputSocket(), psoperation->getInputSocket(0));
+        converter.addLink(scaleOperation->getOutputSocket(), psoperation->getInputSocket(0));
+      }
+      else {
+        // Scale  -> Rotate -> Translate.
+        converter.mapInputSocket(imageInput, scaleOperation->getInputSocket(0));
+
+        converter.addLink(scaleOperation->getOutputSocket(), rotateOperation->getInputSocket(0));
+        converter.addLink(rotateOperation->getOutputSocket(),
+                          translateOperation->getInputSocket(0));
+
+        converter.addLink(translateOperation->getOutputSocket(), psoperation->getInputSocket(0));
+      }
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      TransformOperation *transform_op = new TransformOperation();
+      transform_op->set_sampler(sampler);
+      transform_op->set_convert_rotate_degree_to_rad(false);
+      transform_op->set_invert(invert);
+      converter.addOperation(transform_op);
+      converter.mapInputSocket(imageInput, transform_op->getInputSocket(0));
+      converter.addLink(xAttribute->getOutputSocket(), transform_op->getInputSocket(1));
+      converter.addLink(yAttribute->getOutputSocket(), transform_op->getInputSocket(2));
+      converter.addLink(angleAttribute->getOutputSocket(), transform_op->getInputSocket(3));
+      converter.addLink(scaleAttribute->getOutputSocket(), transform_op->getInputSocket(4));
+      converter.mapOutputSocket(getOutputSocket(), transform_op->getOutputSocket());
+    }
   }
 }
 

source/blender/compositor/nodes/COM_TransformNode.cc

@@ -22,6 +22,7 @@
 #include "COM_ScaleOperation.h"
 #include "COM_SetSamplerOperation.h"
 #include "COM_SetValueOperation.h"
+#include "COM_TransformOperation.h"
 #include "COM_TranslateOperation.h"
 
 namespace blender::compositor {
@@ -32,7 +33,7 @@ TransformNode::TransformNode(bNode *editorNode) : Node(editorNode)
 }
 
 void TransformNode::convertToOperations(NodeConverter &converter,
-                                        const CompositorContext & /*context*/) const
+                                        const CompositorContext &context) const
 {
   NodeInput *imageInput = this->getInputSocket(0);
   NodeInput *xInput = this->getInputSocket(1);
@@ -40,33 +41,51 @@ void TransformNode::convertToOperations(NodeConverter &converter,
   NodeInput *angleInput = this->getInputSocket(3);
   NodeInput *scaleInput = this->getInputSocket(4);
 
-  ScaleRelativeOperation *scaleOperation = new ScaleRelativeOperation();
-  converter.addOperation(scaleOperation);
+  switch (context.get_execution_model()) {
+    case eExecutionModel::Tiled: {
+      ScaleRelativeOperation *scaleOperation = new ScaleRelativeOperation();
+      converter.addOperation(scaleOperation);
 
-  RotateOperation *rotateOperation = new RotateOperation();
-  rotateOperation->setDoDegree2RadConversion(false);
-  converter.addOperation(rotateOperation);
+      RotateOperation *rotateOperation = new RotateOperation();
+      rotateOperation->setDoDegree2RadConversion(false);
+      converter.addOperation(rotateOperation);
 
-  TranslateOperation *translateOperation = new TranslateOperation();
-  converter.addOperation(translateOperation);
+      TranslateOperation *translateOperation = new TranslateOperation();
+      converter.addOperation(translateOperation);
 
-  SetSamplerOperation *sampler = new SetSamplerOperation();
-  sampler->setSampler((PixelSampler)this->getbNode()->custom1);
-  converter.addOperation(sampler);
+      SetSamplerOperation *sampler = new SetSamplerOperation();
+      sampler->setSampler((PixelSampler)this->getbNode()->custom1);
+      converter.addOperation(sampler);
 
-  converter.mapInputSocket(imageInput, sampler->getInputSocket(0));
-  converter.addLink(sampler->getOutputSocket(), scaleOperation->getInputSocket(0));
-  converter.mapInputSocket(scaleInput, scaleOperation->getInputSocket(1));
-  converter.mapInputSocket(scaleInput, scaleOperation->getInputSocket(2));  // xscale = yscale
+      converter.mapInputSocket(imageInput, sampler->getInputSocket(0));
+      converter.addLink(sampler->getOutputSocket(), scaleOperation->getInputSocket(0));
+      converter.mapInputSocket(scaleInput, scaleOperation->getInputSocket(1));
+      converter.mapInputSocket(scaleInput, scaleOperation->getInputSocket(2));  // xscale = yscale
 
-  converter.addLink(scaleOperation->getOutputSocket(), rotateOperation->getInputSocket(0));
-  converter.mapInputSocket(angleInput, rotateOperation->getInputSocket(1));
+      converter.addLink(scaleOperation->getOutputSocket(), rotateOperation->getInputSocket(0));
+      converter.mapInputSocket(angleInput, rotateOperation->getInputSocket(1));
 
-  converter.addLink(rotateOperation->getOutputSocket(), translateOperation->getInputSocket(0));
-  converter.mapInputSocket(xInput, translateOperation->getInputSocket(1));
-  converter.mapInputSocket(yInput, translateOperation->getInputSocket(2));
+      converter.addLink(rotateOperation->getOutputSocket(), translateOperation->getInputSocket(0));
+      converter.mapInputSocket(xInput, translateOperation->getInputSocket(1));
+      converter.mapInputSocket(yInput, translateOperation->getInputSocket(2));
 
-  converter.mapOutputSocket(getOutputSocket(), translateOperation->getOutputSocket());
+      converter.mapOutputSocket(getOutputSocket(), translateOperation->getOutputSocket());
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      TransformOperation *op = new TransformOperation();
+      op->set_sampler((PixelSampler)this->getbNode()->custom1);
+      converter.addOperation(op);
+
+      converter.mapInputSocket(imageInput, op->getInputSocket(0));
+      converter.mapInputSocket(xInput, op->getInputSocket(1));
+      converter.mapInputSocket(yInput, op->getInputSocket(2));
+      converter.mapInputSocket(angleInput, op->getInputSocket(3));
+      converter.mapInputSocket(scaleInput, op->getInputSocket(4));
+      converter.mapOutputSocket(getOutputSocket(), op->getOutputSocket());
+      break;
+    }
+  }
 }
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_BlurBaseOperation.cc

@@ -17,6 +17,8 @@
  */
 
 #include "COM_BlurBaseOperation.h"
+#include "COM_ConstantOperation.h"
+
 #include "BLI_math.h"
 #include "MEM_guardedalloc.h"
 
@@ -36,11 +38,15 @@ BlurBaseOperation::BlurBaseOperation(DataType data_type)
   this->m_size = 1.0f;
   this->m_sizeavailable = false;
   this->m_extend_bounds = false;
+  use_variable_size_ = false;
 }
-void BlurBaseOperation::initExecution()
+
+void BlurBaseOperation::init_data()
 {
-  this->m_inputProgram = this->getInputSocketReader(0);
-  this->m_inputSize = this->getInputSocketReader(1);
+  if (execution_model_ == eExecutionModel::FullFrame) {
+    updateSize();
+  }
+
   this->m_data.image_in_width = this->getWidth();
   this->m_data.image_in_height = this->getHeight();
   if (this->m_data.relative) {
@@ -61,6 +67,12 @@ void BlurBaseOperation::initExecution()
     this->m_data.sizex = round_fl_to_int(this->m_data.percentx * 0.01f * sizex);
     this->m_data.sizey = round_fl_to_int(this->m_data.percenty * 0.01f * sizey);
   }
+}
+
+void BlurBaseOperation::initExecution()
+{
+  this->m_inputProgram = this->getInputSocketReader(0);
+  this->m_inputSize = this->getInputSocketReader(1);
 
   QualityStepHelper::initExecution(COM_QH_MULTIPLY);
 }
@@ -165,23 +177,82 @@ void BlurBaseOperation::setData(const NodeBlurData *data)
   memcpy(&m_data, data, sizeof(NodeBlurData));
 }
 
+int BlurBaseOperation::get_blur_size(eDimension dim) const
+{
+  switch (dim) {
+    case eDimension::X:
+      return m_data.sizex;
+    case eDimension::Y:
+      return m_data.sizey;
+  }
+  return -1;
+}
+
 void BlurBaseOperation::updateSize()
 {
-  if (!this->m_sizeavailable) {
-    float result[4];
-    this->getInputSocketReader(1)->readSampled(result, 0, 0, PixelSampler::Nearest);
-    this->m_size = result[0];
-    this->m_sizeavailable = true;
+  if (this->m_sizeavailable || use_variable_size_) {
+    return;
   }
+
+  switch (execution_model_) {
+    case eExecutionModel::Tiled: {
+      float result[4];
+      this->getInputSocketReader(1)->readSampled(result, 0, 0, PixelSampler::Nearest);
+      this->m_size = result[0];
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      NodeOperation *size_input = get_input_operation(SIZE_INPUT_INDEX);
+      if (size_input->get_flags().is_constant_operation) {
+        m_size = *static_cast<ConstantOperation *>(size_input)->get_constant_elem();
+      } /* Else use default. */
+      break;
+    }
+  }
+  this->m_sizeavailable = true;
 }
 
 void BlurBaseOperation::determineResolution(unsigned int resolution[2],
                                             unsigned int preferredResolution[2])
 {
-  NodeOperation::determineResolution(resolution, preferredResolution);
-  if (this->m_extend_bounds) {
-    resolution[0] += 2 * this->m_size * m_data.sizex;
-    resolution[1] += 2 * this->m_size * m_data.sizey;
+  if (!m_extend_bounds) {
+    NodeOperation::determineResolution(resolution, preferredResolution);
+    return;
+  }
+
+  switch (execution_model_) {
+    case eExecutionModel::Tiled: {
+      NodeOperation::determineResolution(resolution, preferredResolution);
+      resolution[0] += 2 * m_size * m_data.sizex;
+      resolution[1] += 2 * m_size * m_data.sizey;
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      /* Setting a modifier ensures all non main inputs have extended bounds as preferred
+       * resolution, avoiding unnecessary resolution convertions that would hide constant
+       * operations. */
+      set_determined_resolution_modifier([=](unsigned int res[2]) {
+        /* Rounding to even prevents jiggling in backdrop while switching size values. */
+        res[0] += round_to_even(2 * m_size * m_data.sizex);
+        res[1] += round_to_even(2 * m_size * m_data.sizey);
+      });
+      NodeOperation::determineResolution(resolution, preferredResolution);
+      break;
+    }
+  }
+}
+
+void BlurBaseOperation::get_area_of_interest(const int input_idx,
+                                             const rcti &output_area,
+                                             rcti &r_input_area)
+{
+  switch (input_idx) {
+    case 0:
+      r_input_area = output_area;
+      break;
+    case 1:
+      r_input_area = use_variable_size_ ? output_area : COM_SINGLE_ELEM_AREA;
+      break;
   }
 }
 

source/blender/compositor/operations/COM_BlurBaseOperation.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "COM_NodeOperation.h"
+#include "COM_MultiThreadedOperation.h"
 #include "COM_QualityStepHelper.h"
 
 #define MAX_GAUSSTAB_RADIUS 30000
@@ -27,10 +27,16 @@
 
 namespace blender::compositor {
 
-class BlurBaseOperation : public NodeOperation, public QualityStepHelper {
+class BlurBaseOperation : public MultiThreadedOperation, public QualityStepHelper {
  private:
+  bool m_extend_bounds;
+
  protected:
-  BlurBaseOperation(DataType data_type);
+  static constexpr int IMAGE_INPUT_INDEX = 0;
+  static constexpr int SIZE_INPUT_INDEX = 1;
+
+ protected:
+  BlurBaseOperation(DataType data_type8);
   float *make_gausstab(float rad, int size);
 #ifdef BLI_HAVE_SSE2
   __m128 *convert_gausstab_sse(const float *gausstab, int size);
@@ -49,9 +55,11 @@ class BlurBaseOperation : public NodeOperation, public QualityStepHelper {
   float m_size;
   bool m_sizeavailable;
 
-  bool m_extend_bounds;
+  /* Flags for inheriting classes. */
+  bool use_variable_size_;
 
  public:
+  virtual void init_data() override;
   /**
    * Initialize the execution
    */
@@ -75,8 +83,14 @@ class BlurBaseOperation : public NodeOperation, public QualityStepHelper {
     this->m_extend_bounds = extend_bounds;
   }
 
+  int get_blur_size(eDimension dim) const;
+
   void determineResolution(unsigned int resolution[2],
                            unsigned int preferredResolution[2]) override;
+
+  virtual void get_area_of_interest(int input_idx,
+                                    const rcti &output_area,
+                                    rcti &r_input_area) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_BokehBlurOperation.cc

@@ -17,6 +17,8 @@
  */
 
 #include "COM_BokehBlurOperation.h"
+#include "COM_ConstantOperation.h"
+
 #include "BLI_math.h"
 #include "COM_OpenCLDevice.h"
 
@@ -24,6 +26,11 @@
 
 namespace blender::compositor {
 
+constexpr int IMAGE_INPUT_INDEX = 0;
+constexpr int BOKEH_INPUT_INDEX = 1;
+constexpr int BOUNDING_BOX_INPUT_INDEX = 2;
+constexpr int SIZE_INPUT_INDEX = 3;
+
 BokehBlurOperation::BokehBlurOperation()
 {
   this->addInputSocket(DataType::Color);
@@ -44,6 +51,23 @@ BokehBlurOperation::BokehBlurOperation()
   this->m_extend_bounds = false;
 }
 
+void BokehBlurOperation::init_data()
+{
+  if (execution_model_ == eExecutionModel::FullFrame) {
+    updateSize();
+  }
+
+  NodeOperation *bokeh = get_input_operation(BOKEH_INPUT_INDEX);
+  const int width = bokeh->getWidth();
+  const int height = bokeh->getHeight();
+
+  const float dimension = MIN2(width, height);
+
+  m_bokehMidX = width / 2.0f;
+  m_bokehMidY = height / 2.0f;
+  m_bokehDimension = dimension / 2.0f;
+}
+
 void *BokehBlurOperation::initializeTileData(rcti * /*rect*/)
 {
   lockMutex();
@@ -58,18 +82,11 @@ void *BokehBlurOperation::initializeTileData(rcti * /*rect*/)
 void BokehBlurOperation::initExecution()
 {
   initMutex();
+
   this->m_inputProgram = getInputSocketReader(0);
   this->m_inputBokehProgram = getInputSocketReader(1);
   this->m_inputBoundingBoxReader = getInputSocketReader(2);
 
-  int width = this->m_inputBokehProgram->getWidth();
-  int height = this->m_inputBokehProgram->getHeight();
-
-  float dimension = MIN2(width, height);
-
-  this->m_bokehMidX = width / 2.0f;
-  this->m_bokehMidY = height / 2.0f;
-  this->m_bokehDimension = dimension / 2.0f;
   QualityStepHelper::initExecution(COM_QH_INCREASE);
 }
 
@@ -225,23 +242,146 @@ void BokehBlurOperation::executeOpenCL(OpenCLDevice *device,
 
 void BokehBlurOperation::updateSize()
 {
-  if (!this->m_sizeavailable) {
-    float result[4];
-    this->getInputSocketReader(3)->readSampled(result, 0, 0, PixelSampler::Nearest);
-    this->m_size = result[0];
-    CLAMP(this->m_size, 0.0f, 10.0f);
-    this->m_sizeavailable = true;
+  if (this->m_sizeavailable) {
+    return;
   }
+
+  switch (execution_model_) {
+    case eExecutionModel::Tiled: {
+      float result[4];
+      this->getInputSocketReader(3)->readSampled(result, 0, 0, PixelSampler::Nearest);
+      this->m_size = result[0];
+      CLAMP(this->m_size, 0.0f, 10.0f);
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      NodeOperation *size_input = get_input_operation(SIZE_INPUT_INDEX);
+      if (size_input->get_flags().is_constant_operation) {
+        m_size = *static_cast<ConstantOperation *>(size_input)->get_constant_elem();
+        CLAMP(m_size, 0.0f, 10.0f);
+      } /* Else use default. */
+      break;
+    }
+  }
+  this->m_sizeavailable = true;
 }
 
 void BokehBlurOperation::determineResolution(unsigned int resolution[2],
                                              unsigned int preferredResolution[2])
 {
-  NodeOperation::determineResolution(resolution, preferredResolution);
-  if (this->m_extend_bounds) {
-    const float max_dim = MAX2(resolution[0], resolution[1]);
-    resolution[0] += 2 * this->m_size * max_dim / 100.0f;
-    resolution[1] += 2 * this->m_size * max_dim / 100.0f;
+  if (!m_extend_bounds) {
+    NodeOperation::determineResolution(resolution, preferredResolution);
+    return;
+  }
+
+  switch (execution_model_) {
+    case eExecutionModel::Tiled: {
+      NodeOperation::determineResolution(resolution, preferredResolution);
+      const float max_dim = MAX2(resolution[0], resolution[1]);
+      resolution[0] += 2 * this->m_size * max_dim / 100.0f;
+      resolution[1] += 2 * this->m_size * max_dim / 100.0f;
+      break;
+    }
+    case eExecutionModel::FullFrame: {
+      set_determined_resolution_modifier([=](unsigned int res[2]) {
+        const float max_dim = MAX2(res[0], res[1]);
+        /* Rounding to even prevents image jiggling in backdrop while switching size values. */
+        float add_size = round_to_even(2 * this->m_size * max_dim / 100.0f);
+        res[0] += add_size;
+        res[1] += add_size;
+      });
+      NodeOperation::determineResolution(resolution, preferredResolution);
+      break;
+    }
+  }
+}
+
+void BokehBlurOperation::get_area_of_interest(const int input_idx,
+                                              const rcti &output_area,
+                                              rcti &r_input_area)
+{
+  switch (input_idx) {
+    case IMAGE_INPUT_INDEX: {
+      const float max_dim = MAX2(this->getWidth(), this->getHeight());
+      const float add_size = m_size * max_dim / 100.0f;
+      r_input_area.xmin = output_area.xmin - add_size;
+      r_input_area.xmax = output_area.xmax + add_size;
+      r_input_area.ymin = output_area.ymin - add_size;
+      r_input_area.ymax = output_area.ymax + add_size;
+      break;
+    }
+    case BOKEH_INPUT_INDEX: {
+      NodeOperation *bokeh_input = getInputOperation(BOKEH_INPUT_INDEX);
+      r_input_area.xmin = 0;
+      r_input_area.xmax = bokeh_input->getWidth();
+      r_input_area.ymin = 0;
+      r_input_area.ymax = bokeh_input->getHeight();
+      break;
+    }
+    case BOUNDING_BOX_INPUT_INDEX:
+      r_input_area = output_area;
+      break;
+    case SIZE_INPUT_INDEX: {
+      r_input_area = COM_SINGLE_ELEM_AREA;
+      break;
+    }
+  }
+}
+
+void BokehBlurOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                      const rcti &area,
+                                                      Span<MemoryBuffer *> inputs)
+{
+  const float max_dim = MAX2(this->getWidth(), this->getHeight());
+  const int pixel_size = m_size * max_dim / 100.0f;
+  const float m = m_bokehDimension / pixel_size;
+
+  const MemoryBuffer *image_input = inputs[IMAGE_INPUT_INDEX];
+  const MemoryBuffer *bokeh_input = inputs[BOKEH_INPUT_INDEX];
+  MemoryBuffer *bounding_input = inputs[BOUNDING_BOX_INPUT_INDEX];
+  BuffersIterator<float> it = output->iterate_with({bounding_input}, area);
+  const rcti &image_rect = image_input->get_rect();
+  for (; !it.is_end(); ++it) {
+    const int x = it.x;
+    const int y = it.y;
+    const float bounding_box = *it.in(0);
+    if (bounding_box <= 0.0f) {
+      image_input->read_elem(x, y, it.out);
+      continue;
+    }
+
+    float color_accum[4] = {0};
+    float multiplier_accum[4] = {0};
+    if (pixel_size < 2) {
+      image_input->read_elem(x, y, color_accum);
+      multiplier_accum[0] = 1.0f;
+      multiplier_accum[1] = 1.0f;
+      multiplier_accum[2] = 1.0f;
+      multiplier_accum[3] = 1.0f;
+    }
+    const int miny = MAX2(y - pixel_size, image_rect.ymin);
+    const int maxy = MIN2(y + pixel_size, image_rect.ymax);
+    const int minx = MAX2(x - pixel_size, image_rect.xmin);
+    const int maxx = MIN2(x + pixel_size, image_rect.xmax);
+    const int step = getStep();
+    const int elem_stride = image_input->elem_stride * step;
+    const int row_stride = image_input->row_stride * step;
+    const float *row_color = image_input->get_elem(minx, miny);
+    for (int ny = miny; ny < maxy; ny += step, row_color += row_stride) {
+      const float *color = row_color;
+      const float v = m_bokehMidY - (ny - y) * m;
+      for (int nx = minx; nx < maxx; nx += step, color += elem_stride) {
+        const float u = m_bokehMidX - (nx - x) * m;
+        float bokeh[4];
+        bokeh_input->read_elem_checked(u, v, bokeh);
+        madd_v4_v4v4(color_accum, bokeh, color);
+        add_v4_v4(multiplier_accum, bokeh);
+      }
+    }
+    it.out[0] = color_accum[0] * (1.0f / multiplier_accum[0]);
+    it.out[1] = color_accum[1] * (1.0f / multiplier_accum[1]);
+    it.out[2] = color_accum[2] * (1.0f / multiplier_accum[2]);
+    it.out[3] = color_accum[3] * (1.0f / multiplier_accum[3]);
   }
 }
 

source/blender/compositor/operations/COM_BokehBlurOperation.h

@@ -18,12 +18,12 @@
 
 #pragma once
 
-#include "COM_NodeOperation.h"
+#include "COM_MultiThreadedOperation.h"
 #include "COM_QualityStepHelper.h"
 
 namespace blender::compositor {
 
-class BokehBlurOperation : public NodeOperation, public QualityStepHelper {
+class BokehBlurOperation : public MultiThreadedOperation, public QualityStepHelper {
  private:
   SocketReader *m_inputProgram;
   SocketReader *m_inputBokehProgram;
@@ -31,6 +31,7 @@ class BokehBlurOperation : public NodeOperation, public QualityStepHelper {
   void updateSize();
   float m_size;
   bool m_sizeavailable;
+
   float m_bokehMidX;
   float m_bokehMidY;
   float m_bokehDimension;
@@ -39,6 +40,8 @@ class BokehBlurOperation : public NodeOperation, public QualityStepHelper {
  public:
   BokehBlurOperation();
 
+  void init_data() override;
+
   void *initializeTileData(rcti *rect) override;
   /**
    * The inner loop of this operation.
@@ -79,6 +82,11 @@ class BokehBlurOperation : public NodeOperation, public QualityStepHelper {
 
   void determineResolution(unsigned int resolution[2],
                            unsigned int preferredResolution[2]) override;
+
+  void get_area_of_interest(int input_idx, const rcti &output_area, rcti &r_input_area) override;
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ChannelMatteOperation.cc

@@ -27,6 +27,7 @@ ChannelMatteOperation::ChannelMatteOperation()
   addOutputSocket(DataType::Value);
 
   this->m_inputImageProgram = nullptr;
+  flags.can_be_constant = true;
 }
 
 void ChannelMatteOperation::initExecution()
@@ -121,4 +122,37 @@ void ChannelMatteOperation::executePixelSampled(float output[4],
   output[0] = MIN2(alpha, inColor[3]);
 }
 
+void ChannelMatteOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                         const rcti &area,
+                                                         Span<MemoryBuffer *> inputs)
+{
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    const float *color = it.in(0);
+
+    /* Matte operation. */
+    float alpha = color[this->m_ids[0]] - MAX2(color[this->m_ids[1]], color[this->m_ids[2]]);
+
+    /* Flip because 0.0 is transparent, not 1.0. */
+    alpha = 1.0f - alpha;
+
+    /* Test range. */
+    if (alpha > m_limit_max) {
+      alpha = color[3]; /* Whatever it was prior. */
+    }
+    else if (alpha < m_limit_min) {
+      alpha = 0.0f;
+    }
+    else { /* Blend. */
+      alpha = (alpha - m_limit_min) / m_limit_range;
+    }
+
+    /* Store matte(alpha) value in [0] to go with
+     * COM_SetAlphaMultiplyOperation and the Value output.
+     */
+
+    /* Don't make something that was more transparent less transparent. */
+    *it.out = MIN2(alpha, color[3]);
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ChannelMatteOperation.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "COM_MixOperation.h"
+#include "COM_MultiThreadedOperation.h"
 
 namespace blender::compositor {
 
@@ -26,7 +26,7 @@ namespace blender::compositor {
  * this program converts an input color to an output value.
  * it assumes we are in sRGB color space.
  */
-class ChannelMatteOperation : public NodeOperation {
+class ChannelMatteOperation : public MultiThreadedOperation {
  private:
   SocketReader *m_inputImageProgram;
 
@@ -71,6 +71,10 @@ class ChannelMatteOperation : public NodeOperation {
     this->m_limit_channel = nodeChroma->channel;
     this->m_matte_channel = custom2;
   }
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ChromaMatteOperation.cc

@@ -29,6 +29,7 @@ ChromaMatteOperation::ChromaMatteOperation()
 
   this->m_inputImageProgram = nullptr;
   this->m_inputKeyProgram = nullptr;
+  flags.can_be_constant = true;
 }
 
 void ChromaMatteOperation::initExecution()
@@ -110,4 +111,58 @@ void ChromaMatteOperation::executePixelSampled(float output[4],
   }
 }
 
+void ChromaMatteOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                        const rcti &area,
+                                                        Span<MemoryBuffer *> inputs)
+{
+  const float acceptance = this->m_settings->t1; /* In radians. */
+  const float cutoff = this->m_settings->t2;     /* In radians. */
+  const float gain = this->m_settings->fstrength;
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    const float *in_image = it.in(0);
+    const float *in_key = it.in(1);
+
+    /* Store matte(alpha) value in [0] to go with
+     * #COM_SetAlphaMultiplyOperation and the Value output. */
+
+    /* Algorithm from book "Video Demystified", does not include the spill reduction part. */
+    /* Find theta, the angle that the color space should be rotated based on key. */
+
+    /* Rescale to `-1.0..1.0`. */
+    // const float image_Y = (in_image[0] * 2.0f) - 1.0f;  // UNUSED
+    const float image_cb = (in_image[1] * 2.0f) - 1.0f;
+    const float image_cr = (in_image[2] * 2.0f) - 1.0f;
+
+    // const float key_Y = (in_key[0] * 2.0f) - 1.0f;  // UNUSED
+    const float key_cb = (in_key[1] * 2.0f) - 1.0f;
+    const float key_cr = (in_key[2] * 2.0f) - 1.0f;
+
+    const float theta = atan2(key_cr, key_cb);
+
+    /* Rotate the cb and cr into x/z space. */
+    const float x_angle = image_cb * cosf(theta) + image_cr * sinf(theta);
+    const float z_angle = image_cr * cosf(theta) - image_cb * sinf(theta);
+
+    /* If within the acceptance angle. */
+    /* If kfg is <0 then the pixel is outside of the key color. */
+    const float kfg = x_angle - (fabsf(z_angle) / tanf(acceptance / 2.0f));
+
+    if (kfg > 0.0f) { /* Found a pixel that is within key color. */
+      const float beta = atan2(z_angle, x_angle);
+      float alpha = 1.0f - (kfg / gain);
+
+      /* Ff beta is within the cutoff angle. */
+      if (fabsf(beta) < (cutoff / 2.0f)) {
+        alpha = 0.0f;
+      }
+
+      /* Don't make something that was more transparent less transparent. */
+      it.out[0] = alpha < in_image[3] ? alpha : in_image[3];
+    }
+    else {                     /* Pixel is outside key color. */
+      it.out[0] = in_image[3]; /* Make pixel just as transparent as it was before. */
+    }
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ChromaMatteOperation.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "COM_MixOperation.h"
+#include "COM_MultiThreadedOperation.h"
 
 namespace blender::compositor {
 
@@ -26,7 +26,7 @@ namespace blender::compositor {
  * this program converts an input color to an output value.
  * it assumes we are in sRGB color space.
  */
-class ChromaMatteOperation : public NodeOperation {
+class ChromaMatteOperation : public MultiThreadedOperation {
  private:
   NodeChroma *m_settings;
   SocketReader *m_inputImageProgram;
@@ -50,6 +50,10 @@ class ChromaMatteOperation : public NodeOperation {
   {
     this->m_settings = nodeChroma;
   }
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ColorMatteOperation.cc

@@ -29,6 +29,7 @@ ColorMatteOperation::ColorMatteOperation()
 
   this->m_inputImageProgram = nullptr;
   this->m_inputKeyProgram = nullptr;
+  flags.can_be_constant = true;
 }
 
 void ColorMatteOperation::initExecution()
@@ -82,4 +83,40 @@ void ColorMatteOperation::executePixelSampled(float output[4],
   }
 }
 
+void ColorMatteOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                       const rcti &area,
+                                                       Span<MemoryBuffer *> inputs)
+{
+  const float hue = m_settings->t1;
+  const float sat = m_settings->t2;
+  const float val = m_settings->t3;
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    const float *in_color = it.in(0);
+    const float *in_key = it.in(1);
+
+    /* Store matte(alpha) value in [0] to go with
+     * COM_SetAlphaMultiplyOperation and the Value output.
+     */
+
+    float h_wrap;
+    if (
+        /* Do hue last because it needs to wrap, and does some more checks. */
+
+        /* #sat */ (fabsf(in_color[1] - in_key[1]) < sat) &&
+        /* #val */ (fabsf(in_color[2] - in_key[2]) < val) &&
+
+        /* Multiply by 2 because it wraps on both sides of the hue,
+         * otherwise 0.5 would key all hue's. */
+
+        /* #hue */
+        ((h_wrap = 2.0f * fabsf(in_color[0] - in_key[0])) < hue || (2.0f - h_wrap) < hue)) {
+      it.out[0] = 0.0f; /* Make transparent. */
+    }
+
+    else {                     /* Pixel is outside key color. */
+      it.out[0] = in_color[3]; /* Make pixel just as transparent as it was before. */
+    }
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ColorMatteOperation.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "COM_MixOperation.h"
+#include "COM_MultiThreadedOperation.h"
 
 namespace blender::compositor {
 
@@ -26,7 +26,7 @@ namespace blender::compositor {
  * this program converts an input color to an output value.
  * it assumes we are in sRGB color space.
  */
-class ColorMatteOperation : public NodeOperation {
+class ColorMatteOperation : public MultiThreadedOperation {
  private:
   NodeChroma *m_settings;
   SocketReader *m_inputImageProgram;
@@ -50,6 +50,10 @@ class ColorMatteOperation : public NodeOperation {
   {
     this->m_settings = nodeChroma;
   }
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ColorRampOperation.cc

@@ -29,6 +29,7 @@ ColorRampOperation::ColorRampOperation()
 
   this->m_inputProgram = nullptr;
   this->m_colorBand = nullptr;
+  this->flags.can_be_constant = true;
 }
 void ColorRampOperation::initExecution()
 {
@@ -51,4 +52,13 @@ void ColorRampOperation::deinitExecution()
   this->m_inputProgram = nullptr;
 }
 
+void ColorRampOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                      const rcti &area,
+                                                      Span<MemoryBuffer *> inputs)
+{
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    BKE_colorband_evaluate(m_colorBand, *it.in(0), it.out);
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ColorRampOperation.h

@@ -18,12 +18,12 @@
 
 #pragma once
 
-#include "COM_NodeOperation.h"
+#include "COM_MultiThreadedOperation.h"
 #include "DNA_texture_types.h"
 
 namespace blender::compositor {
 
-class ColorRampOperation : public NodeOperation {
+class ColorRampOperation : public MultiThreadedOperation {
  private:
   /**
    * Cached reference to the inputProgram
@@ -53,6 +53,10 @@ class ColorRampOperation : public NodeOperation {
   {
     this->m_colorBand = colorBand;
   }
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ColorSpillOperation.cc

@@ -32,6 +32,7 @@ ColorSpillOperation::ColorSpillOperation()
   this->m_inputFacReader = nullptr;
   this->m_spillChannel = 1;  // GREEN
   this->m_spillMethod = 0;
+  flags.can_be_constant = true;
 }
 
 void ColorSpillOperation::initExecution()
@@ -118,4 +119,36 @@ void ColorSpillOperation::executePixelSampled(float output[4],
   }
 }
 
+void ColorSpillOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                       const rcti &area,
+                                                       Span<MemoryBuffer *> inputs)
+{
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    const float *color = it.in(0);
+    const float factor = MIN2(1.0f, *it.in(1));
+
+    float map;
+    switch (m_spillMethod) {
+      case 0: /* simple */
+        map = factor *
+              (color[m_spillChannel] - (m_settings->limscale * color[m_settings->limchan]));
+        break;
+      default: /* average */
+        map = factor * (color[m_spillChannel] -
+                        (m_settings->limscale * AVG(color[m_channel2], color[m_channel3])));
+        break;
+    }
+
+    if (map > 0.0f) {
+      it.out[0] = color[0] + m_rmut * (m_settings->uspillr * map);
+      it.out[1] = color[1] + m_gmut * (m_settings->uspillg * map);
+      it.out[2] = color[2] + m_bmut * (m_settings->uspillb * map);
+      it.out[3] = color[3];
+    }
+    else {
+      copy_v4_v4(it.out, color);
+    }
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ColorSpillOperation.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "COM_NodeOperation.h"
+#include "COM_MultiThreadedOperation.h"
 
 namespace blender::compositor {
 
@@ -26,7 +26,7 @@ namespace blender::compositor {
  * this program converts an input color to an output value.
  * it assumes we are in sRGB color space.
  */
-class ColorSpillOperation : public NodeOperation {
+class ColorSpillOperation : public MultiThreadedOperation {
  protected:
   NodeColorspill *m_settings;
   SocketReader *m_inputImageReader;
@@ -65,6 +65,10 @@ class ColorSpillOperation : public NodeOperation {
   }
 
   float calculateMapValue(float fac, float *input);
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ConvertOperation.cc

@@ -27,6 +27,7 @@ namespace blender::compositor {
 ConvertBaseOperation::ConvertBaseOperation()
 {
   this->m_inputOperation = nullptr;
+  this->flags.can_be_constant = true;
 }
 
 void ConvertBaseOperation::initExecution()
@@ -39,6 +40,14 @@ void ConvertBaseOperation::deinitExecution()
   this->m_inputOperation = nullptr;
 }
 
+void ConvertBaseOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                        const rcti &area,
+                                                        Span<MemoryBuffer *> inputs)
+{
+  BuffersIterator<float> it = output->iterate_with(inputs, area);
+  update_memory_buffer_partial(it);
+}
+
 /* ******** Value to Color ******** */
 
 ConvertValueToColorOperation::ConvertValueToColorOperation() : ConvertBaseOperation()
@@ -58,6 +67,14 @@ void ConvertValueToColorOperation::executePixelSampled(float output[4],
   output[3] = 1.0f;
 }
 
+void ConvertValueToColorOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    it.out[0] = it.out[1] = it.out[2] = *it.in(0);
+    it.out[3] = 1.0f;
+  }
+}
+
 /* ******** Color to Value ******** */
 
 ConvertColorToValueOperation::ConvertColorToValueOperation() : ConvertBaseOperation()
@@ -76,6 +93,14 @@ void ConvertColorToValueOperation::executePixelSampled(float output[4],
   output[0] = (inputColor[0] + inputColor[1] + inputColor[2]) / 3.0f;
 }
 
+void ConvertColorToValueOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    it.out[0] = (in[0] + in[1] + in[2]) / 3.0f;
+  }
+}
+
 /* ******** Color to BW ******** */
 
 ConvertColorToBWOperation::ConvertColorToBWOperation() : ConvertBaseOperation()
@@ -94,6 +119,13 @@ void ConvertColorToBWOperation::executePixelSampled(float output[4],
   output[0] = IMB_colormanagement_get_luminance(inputColor);
 }
 
+void ConvertColorToBWOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    it.out[0] = IMB_colormanagement_get_luminance(it.in(0));
+  }
+}
+
 /* ******** Color to Vector ******** */
 
 ConvertColorToVectorOperation::ConvertColorToVectorOperation() : ConvertBaseOperation()
@@ -112,6 +144,13 @@ void ConvertColorToVectorOperation::executePixelSampled(float output[4],
   copy_v3_v3(output, color);
 }
 
+void ConvertColorToVectorOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    copy_v3_v3(it.out, it.in(0));
+  }
+}
+
 /* ******** Value to Vector ******** */
 
 ConvertValueToVectorOperation::ConvertValueToVectorOperation() : ConvertBaseOperation()
@@ -130,6 +169,13 @@ void ConvertValueToVectorOperation::executePixelSampled(float output[4],
   output[0] = output[1] = output[2] = value;
 }
 
+void ConvertValueToVectorOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    it.out[0] = it.out[1] = it.out[2] = *it.in(0);
+  }
+}
+
 /* ******** Vector to Color ******** */
 
 ConvertVectorToColorOperation::ConvertVectorToColorOperation() : ConvertBaseOperation()
@@ -147,6 +193,14 @@ void ConvertVectorToColorOperation::executePixelSampled(float output[4],
   output[3] = 1.0f;
 }
 
+void ConvertVectorToColorOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    copy_v3_v3(it.out, it.in(0));
+    it.out[3] = 1.0f;
+  }
+}
+
 /* ******** Vector to Value ******** */
 
 ConvertVectorToValueOperation::ConvertVectorToValueOperation() : ConvertBaseOperation()
@@ -165,6 +219,14 @@ void ConvertVectorToValueOperation::executePixelSampled(float output[4],
   output[0] = (input[0] + input[1] + input[2]) / 3.0f;
 }
 
+void ConvertVectorToValueOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    it.out[0] = (in[0] + in[1] + in[2]) / 3.0f;
+  }
+}
+
 /* ******** RGB to YCC ******** */
 
 ConvertRGBToYCCOperation::ConvertRGBToYCCOperation() : ConvertBaseOperation()
@@ -207,6 +269,18 @@ void ConvertRGBToYCCOperation::executePixelSampled(float output[4],
   output[3] = inputColor[3];
 }
 
+void ConvertRGBToYCCOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    rgb_to_ycc(in[0], in[1], in[2], &it.out[0], &it.out[1], &it.out[2], this->m_mode);
+
+    /* Normalize for viewing (#rgb_to_ycc returns 0-255 values). */
+    mul_v3_fl(it.out, 1.0f / 255.0f);
+    it.out[3] = in[3];
+  }
+}
+
 /* ******** YCC to RGB ******** */
 
 ConvertYCCToRGBOperation::ConvertYCCToRGBOperation() : ConvertBaseOperation()
@@ -253,6 +327,22 @@ void ConvertYCCToRGBOperation::executePixelSampled(float output[4],
   output[3] = inputColor[3];
 }
 
+void ConvertYCCToRGBOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    /* Multiply by 255 to un-normalize (#ycc_to_rgb needs input values in 0-255 range). */
+    ycc_to_rgb(in[0] * 255.0f,
+               in[1] * 255.0f,
+               in[2] * 255.0f,
+               &it.out[0],
+               &it.out[1],
+               &it.out[2],
+               this->m_mode);
+    it.out[3] = in[3];
+  }
+}
+
 /* ******** RGB to YUV ******** */
 
 ConvertRGBToYUVOperation::ConvertRGBToYUVOperation() : ConvertBaseOperation()
@@ -278,6 +368,15 @@ void ConvertRGBToYUVOperation::executePixelSampled(float output[4],
   output[3] = inputColor[3];
 }
 
+void ConvertRGBToYUVOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    rgb_to_yuv(in[0], in[1], in[2], &it.out[0], &it.out[1], &it.out[2], BLI_YUV_ITU_BT709);
+    it.out[3] = in[3];
+  }
+}
+
 /* ******** YUV to RGB ******** */
 
 ConvertYUVToRGBOperation::ConvertYUVToRGBOperation() : ConvertBaseOperation()
@@ -303,6 +402,15 @@ void ConvertYUVToRGBOperation::executePixelSampled(float output[4],
   output[3] = inputColor[3];
 }
 
+void ConvertYUVToRGBOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    yuv_to_rgb(in[0], in[1], in[2], &it.out[0], &it.out[1], &it.out[2], BLI_YUV_ITU_BT709);
+    it.out[3] = in[3];
+  }
+}
+
 /* ******** RGB to HSV ******** */
 
 ConvertRGBToHSVOperation::ConvertRGBToHSVOperation() : ConvertBaseOperation()
@@ -322,6 +430,15 @@ void ConvertRGBToHSVOperation::executePixelSampled(float output[4],
   output[3] = inputColor[3];
 }
 
+void ConvertRGBToHSVOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    rgb_to_hsv_v(in, it.out);
+    it.out[3] = in[3];
+  }
+}
+
 /* ******** HSV to RGB ******** */
 
 ConvertHSVToRGBOperation::ConvertHSVToRGBOperation() : ConvertBaseOperation()
@@ -344,6 +461,18 @@ void ConvertHSVToRGBOperation::executePixelSampled(float output[4],
   output[3] = inputColor[3];
 }
 
+void ConvertHSVToRGBOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    const float *in = it.in(0);
+    hsv_to_rgb_v(in, it.out);
+    it.out[0] = max_ff(it.out[0], 0.0f);
+    it.out[1] = max_ff(it.out[1], 0.0f);
+    it.out[2] = max_ff(it.out[2], 0.0f);
+    it.out[3] = in[3];
+  }
+}
+
 /* ******** Premul to Straight ******** */
 
 ConvertPremulToStraightOperation::ConvertPremulToStraightOperation() : ConvertBaseOperation()
@@ -363,6 +492,13 @@ void ConvertPremulToStraightOperation::executePixelSampled(float output[4],
   copy_v4_v4(output, converted);
 }
 
+void ConvertPremulToStraightOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    copy_v4_v4(it.out, ColorSceneLinear4f<eAlpha::Premultiplied>(it.in(0)).unpremultiply_alpha());
+  }
+}
+
 /* ******** Straight to Premul ******** */
 
 ConvertStraightToPremulOperation::ConvertStraightToPremulOperation() : ConvertBaseOperation()
@@ -382,6 +518,13 @@ void ConvertStraightToPremulOperation::executePixelSampled(float output[4],
   copy_v4_v4(output, converted);
 }
 
+void ConvertStraightToPremulOperation::update_memory_buffer_partial(BuffersIterator<float> &it)
+{
+  for (; !it.is_end(); ++it) {
+    copy_v4_v4(it.out, ColorSceneLinear4f<eAlpha::Straight>(it.in(0)).premultiply_alpha());
+  }
+}
+
 /* ******** Separate Channels ******** */
 
 SeparateChannelOperation::SeparateChannelOperation()
@@ -410,6 +553,15 @@ void SeparateChannelOperation::executePixelSampled(float output[4],
   output[0] = input[this->m_channel];
 }
 
+void SeparateChannelOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                            const rcti &area,
+                                                            Span<MemoryBuffer *> inputs)
+{
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    it.out[0] = it.in(0)[this->m_channel];
+  }
+}
+
 /* ******** Combine Channels ******** */
 
 CombineChannelsOperation::CombineChannelsOperation()
@@ -466,4 +618,16 @@ void CombineChannelsOperation::executePixelSampled(float output[4],
   }
 }
 
+void CombineChannelsOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                            const rcti &area,
+                                                            Span<MemoryBuffer *> inputs)
+{
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    it.out[0] = *it.in(0);
+    it.out[1] = *it.in(1);
+    it.out[2] = *it.in(2);
+    it.out[3] = *it.in(3);
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_ConvertOperation.h

@@ -18,11 +18,11 @@
 
 #pragma once
 
-#include "COM_NodeOperation.h"
+#include "COM_MultiThreadedOperation.h"
 
 namespace blender::compositor {
 
-class ConvertBaseOperation : public NodeOperation {
+class ConvertBaseOperation : public MultiThreadedOperation {
  protected:
   SocketReader *m_inputOperation;
 
@@ -31,6 +31,13 @@ class ConvertBaseOperation : public NodeOperation {
 
   void initExecution() override;
   void deinitExecution() override;
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) final;
+
+ protected:
+  virtual void update_memory_buffer_partial(BuffersIterator<float> &it) = 0;
 };
 
 class ConvertValueToColorOperation : public ConvertBaseOperation {
@@ -38,6 +45,9 @@ class ConvertValueToColorOperation : public ConvertBaseOperation {
   ConvertValueToColorOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertColorToValueOperation : public ConvertBaseOperation {
@@ -45,6 +55,9 @@ class ConvertColorToValueOperation : public ConvertBaseOperation {
   ConvertColorToValueOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertColorToBWOperation : public ConvertBaseOperation {
@@ -52,6 +65,9 @@ class ConvertColorToBWOperation : public ConvertBaseOperation {
   ConvertColorToBWOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertColorToVectorOperation : public ConvertBaseOperation {
@@ -59,6 +75,9 @@ class ConvertColorToVectorOperation : public ConvertBaseOperation {
   ConvertColorToVectorOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertValueToVectorOperation : public ConvertBaseOperation {
@@ -66,6 +85,9 @@ class ConvertValueToVectorOperation : public ConvertBaseOperation {
   ConvertValueToVectorOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertVectorToColorOperation : public ConvertBaseOperation {
@@ -73,6 +95,9 @@ class ConvertVectorToColorOperation : public ConvertBaseOperation {
   ConvertVectorToColorOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertVectorToValueOperation : public ConvertBaseOperation {
@@ -80,6 +105,9 @@ class ConvertVectorToValueOperation : public ConvertBaseOperation {
   ConvertVectorToValueOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertRGBToYCCOperation : public ConvertBaseOperation {
@@ -94,6 +122,9 @@ class ConvertRGBToYCCOperation : public ConvertBaseOperation {
 
   /** Set the YCC mode */
   void setMode(int mode);
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertYCCToRGBOperation : public ConvertBaseOperation {
@@ -108,6 +139,9 @@ class ConvertYCCToRGBOperation : public ConvertBaseOperation {
 
   /** Set the YCC mode */
   void setMode(int mode);
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertRGBToYUVOperation : public ConvertBaseOperation {
@@ -115,6 +149,9 @@ class ConvertRGBToYUVOperation : public ConvertBaseOperation {
   ConvertRGBToYUVOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertYUVToRGBOperation : public ConvertBaseOperation {
@@ -122,6 +159,9 @@ class ConvertYUVToRGBOperation : public ConvertBaseOperation {
   ConvertYUVToRGBOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertRGBToHSVOperation : public ConvertBaseOperation {
@@ -129,6 +169,9 @@ class ConvertRGBToHSVOperation : public ConvertBaseOperation {
   ConvertRGBToHSVOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertHSVToRGBOperation : public ConvertBaseOperation {
@@ -136,6 +179,9 @@ class ConvertHSVToRGBOperation : public ConvertBaseOperation {
   ConvertHSVToRGBOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertPremulToStraightOperation : public ConvertBaseOperation {
@@ -143,6 +189,9 @@ class ConvertPremulToStraightOperation : public ConvertBaseOperation {
   ConvertPremulToStraightOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
 class ConvertStraightToPremulOperation : public ConvertBaseOperation {
@@ -150,9 +199,12 @@ class ConvertStraightToPremulOperation : public ConvertBaseOperation {
   ConvertStraightToPremulOperation();
 
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+ protected:
+  void update_memory_buffer_partial(BuffersIterator<float> &it) override;
 };
 
-class SeparateChannelOperation : public NodeOperation {
+class SeparateChannelOperation : public MultiThreadedOperation {
  private:
   SocketReader *m_inputOperation;
   int m_channel;
@@ -168,9 +220,13 @@ class SeparateChannelOperation : public NodeOperation {
   {
     this->m_channel = channel;
   }
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
-class CombineChannelsOperation : public NodeOperation {
+class CombineChannelsOperation : public MultiThreadedOperation {
  private:
   SocketReader *m_inputChannel1Operation;
   SocketReader *m_inputChannel2Operation;
@@ -183,6 +239,10 @@ class CombineChannelsOperation : public NodeOperation {
 
   void initExecution() override;
   void deinitExecution() override;
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_CropOperation.cc

@@ -95,6 +95,22 @@ void CropOperation::executePixelSampled(float output[4], float x, float y, Pixel
   }
 }
 
+void CropOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                 const rcti &area,
+                                                 Span<MemoryBuffer *> inputs)
+{
+  rcti crop_area;
+  BLI_rcti_init(&crop_area, m_xmin, m_xmax, m_ymin, m_ymax);
+  for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
+    if (BLI_rcti_isect_pt(&crop_area, it.x, it.y)) {
+      copy_v4_v4(it.out, it.in(0));
+    }
+    else {
+      zero_v4(it.out);
+    }
+  }
+}
+
 CropImageOperation::CropImageOperation() : CropBaseOperation()
 {
   /* pass */
@@ -114,6 +130,18 @@ bool CropImageOperation::determineDependingAreaOfInterest(rcti *input,
   return NodeOperation::determineDependingAreaOfInterest(&newInput, readOperation, output);
 }
 
+void CropImageOperation::get_area_of_interest(const int input_idx,
+                                              const rcti &output_area,
+                                              rcti &r_input_area)
+{
+  BLI_assert(input_idx == 0);
+  UNUSED_VARS_NDEBUG(input_idx);
+  r_input_area.xmax = output_area.xmax + this->m_xmin;
+  r_input_area.xmin = output_area.xmin + this->m_xmin;
+  r_input_area.ymax = output_area.ymax + this->m_ymin;
+  r_input_area.ymin = output_area.ymin + this->m_ymin;
+}
+
 void CropImageOperation::determineResolution(unsigned int resolution[2],
                                              unsigned int preferredResolution[2])
 {
@@ -136,4 +164,21 @@ void CropImageOperation::executePixelSampled(float output[4],
   }
 }
 
+void CropImageOperation::update_memory_buffer_partial(MemoryBuffer *output,
+                                                      const rcti &area,
+                                                      Span<MemoryBuffer *> inputs)
+{
+  rcti op_area;
+  BLI_rcti_init(&op_area, 0, getWidth(), 0, getHeight());
+  const MemoryBuffer *input = inputs[0];
+  for (BuffersIterator<float> it = output->iterate_with({}, area); !it.is_end(); ++it) {
+    if (BLI_rcti_isect_pt(&op_area, it.x, it.y)) {
+      input->read_elem_checked(it.x + this->m_xmin, it.y + this->m_ymin, it.out);
+    }
+    else {
+      zero_v4(it.out);
+    }
+  }
+}
+
 }  // namespace blender::compositor

source/blender/compositor/operations/COM_CropOperation.h

@@ -18,11 +18,11 @@
 
 #pragma once
 
-#include "COM_NodeOperation.h"
+#include "COM_MultiThreadedOperation.h"
 
 namespace blender::compositor {
 
-class CropBaseOperation : public NodeOperation {
+class CropBaseOperation : public MultiThreadedOperation {
  protected:
   SocketReader *m_inputOperation;
   NodeTwoXYs *m_settings;
@@ -53,6 +53,10 @@ class CropOperation : public CropBaseOperation {
  public:
   CropOperation();
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 class CropImageOperation : public CropBaseOperation {
@@ -65,6 +69,11 @@ class CropImageOperation : public CropBaseOperation {
   void determineResolution(unsigned int resolution[2],
                            unsigned int preferredResolution[2]) override;
   void executePixelSampled(float output[4], float x, float y, PixelSampler sampler) override;
+
+  void get_area_of_interest(int input_idx, const rcti &output_area, rcti &r_input_area) override;
+  void update_memory_buffer_partial(MemoryBuffer *output,
+                                    const rcti &area,
+                                    Span<MemoryBuffer *> inputs) override;
 };
 
 }  // namespace blender::compositor