intern/cycles/blender/CMakeLists.txt

@@ -114,6 +114,9 @@ endif()
 
 if(WITH_OPENVDB)
   add_definitions(-DWITH_OPENVDB ${OPENVDB_DEFINITIONS})
+  list(APPEND INC
+    ../../../intern/openvdb
+  )
   list(APPEND INC_SYS
     ${OPENVDB_INCLUDE_DIRS}
   )

intern/cycles/blender/volume.cpp

@@ -11,11 +11,8 @@
 #include "blender/sync.h"
 #include "blender/util.h"
 
-#ifdef WITH_OPENVDB
-#  include <openvdb/openvdb.h>
-openvdb::GridBase::ConstPtr BKE_volume_grid_openvdb_for_read(const struct Volume *volume,
-                                                             const struct VolumeGrid *grid);
-#endif
+#include "BKE_volume.hh"
+#include "BKE_volume_grid.hh"
 
 CCL_NAMESPACE_BEGIN
 
@@ -232,16 +229,10 @@ class BlenderVolumeLoader : public VDBImageLoader {
 #ifdef WITH_OPENVDB
     for (BL::VolumeGrid &b_volume_grid : b_volume.grids) {
       if (b_volume_grid.name() == grid_name) {
-        const bool unload = !b_volume_grid.is_loaded();
-
-        ::Volume *volume = (::Volume *)b_volume.ptr.data;
-        const VolumeGrid *volume_grid = (VolumeGrid *)b_volume_grid.ptr.data;
-        grid = BKE_volume_grid_openvdb_for_read(volume, volume_grid);
-
-        if (unload) {
-          b_volume_grid.unload();
-        }
-
+        const auto *volume_grid = static_cast<const blender::bke::VolumeGridData *>(
+            b_volume_grid.ptr.data);
+        tree_access_token = volume_grid->tree_access_token();
+        grid = volume_grid->grid_ptr(tree_access_token);
         break;
       }
     }
@@ -265,6 +256,10 @@ class BlenderVolumeLoader : public VDBImageLoader {
   }
 
   BL::Volume b_volume;
+#ifdef WITH_OPENVDB
+  /* Store tree user so that the openvdb grid that is shared with Blender is not unloaded. */
+  blender::bke::VolumeTreeAccessToken tree_access_token;
+#endif
 };
 
 static void sync_volume_object(BL::BlendData &b_data,

intern/openvdb/CMakeLists.txt

@@ -11,6 +11,7 @@ set(INC_SYS
 
 set(SRC
   openvdb_capi.h
+  openvdb_fwd.hh
 )
 
 set(LIB

intern/openvdb/openvdb_fwd.hh

@@ -0,0 +1,111 @@
+/* SPDX-FileCopyrightText: 2023 Blender Foundation
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#pragma once
+
+/* Note: version header included here to enable correct forward declaration of some types. No other
+ * OpenVDB headers should be included here, especially openvdb.h, to avoid affecting other
+ * compilation units. */
+#include <openvdb/Types.h>
+#include <openvdb/version.h>
+
+/* -------------------------------------------------------------------- */
+/** \name OpenVDB Forward Declaration
+ * \{ */
+
+/* Forward declaration for basic OpenVDB types. */
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+
+class GridBase;
+class MetaMap;
+template<typename TreeType> class Grid;
+
+namespace math {
+class Transform;
+}
+
+namespace tree {
+class TreeBase;
+template<typename T, Index Log2Dim> class LeafNode;
+template<typename ChildNodeType, Index Log2Dim> class InternalNode;
+template<typename ChildNodeType> class RootNode;
+template<typename RootNodeType> class Tree;
+
+/* Forward-declared version of Tree4, can't use the actual Tree4 alias because it can't be
+ * forward-declared. */
+template<typename T, Index N1 = 5, Index N2 = 4, Index N3 = 3> struct Tree4Fwd {
+  using Type = openvdb::tree::Tree<openvdb::tree::RootNode<
+      openvdb::tree::InternalNode<openvdb::tree::InternalNode<openvdb::tree::LeafNode<T, N3>, N2>,
+                                  N1>>>;
+};
+}  // namespace tree
+
+namespace tools {
+template<typename T, Index Log2Dim> struct PointIndexLeafNode;
+using PointIndexTree = tree::Tree<tree::RootNode<
+    tree::InternalNode<tree::InternalNode<PointIndexLeafNode<PointIndex32, 3>, 4>, 5>>>;
+using PointIndexGrid = Grid<PointIndexTree>;
+}  // namespace tools
+
+namespace points {
+template<typename T, Index Log2Dim> class PointDataLeafNode;
+using PointDataTree = tree::Tree<tree::RootNode<
+    tree::InternalNode<tree::InternalNode<PointDataLeafNode<PointDataIndex32, 3>, 4>, 5>>>;
+using PointDataGrid = Grid<PointDataTree>;
+struct NullCodec;
+template<typename ValueType, typename Codec> class TypedAttributeArray;
+}  // namespace points
+
+/// Common tree types
+using BoolTree = tree::Tree4Fwd<bool, 5, 4, 3>::Type;
+using DoubleTree = tree::Tree4Fwd<double, 5, 4, 3>::Type;
+using FloatTree = tree::Tree4Fwd<float, 5, 4, 3>::Type;
+using Int8Tree = tree::Tree4Fwd<int8_t, 5, 4, 3>::Type;
+using Int32Tree = tree::Tree4Fwd<int32_t, 5, 4, 3>::Type;
+using Int64Tree = tree::Tree4Fwd<int64_t, 5, 4, 3>::Type;
+using MaskTree = tree::Tree4Fwd<ValueMask, 5, 4, 3>::Type;
+using UInt32Tree = tree::Tree4Fwd<uint32_t, 5, 4, 3>::Type;
+using Vec2DTree = tree::Tree4Fwd<Vec2d, 5, 4, 3>::Type;
+using Vec2ITree = tree::Tree4Fwd<Vec2i, 5, 4, 3>::Type;
+using Vec2STree = tree::Tree4Fwd<Vec2s, 5, 4, 3>::Type;
+using Vec3DTree = tree::Tree4Fwd<Vec3d, 5, 4, 3>::Type;
+using Vec3ITree = tree::Tree4Fwd<Vec3i, 5, 4, 3>::Type;
+using Vec3STree = tree::Tree4Fwd<Vec3f, 5, 4, 3>::Type;
+using Vec4STree = tree::Tree4Fwd<Vec4f, 5, 4, 3>::Type;
+using ScalarTree = FloatTree;
+using TopologyTree = MaskTree;
+using Vec3dTree = Vec3DTree;
+using Vec3fTree = Vec3STree;
+using Vec4fTree = Vec4STree;
+using VectorTree = Vec3fTree;
+
+/// Common grid types
+using BoolGrid = Grid<BoolTree>;
+using DoubleGrid = Grid<DoubleTree>;
+using FloatGrid = Grid<FloatTree>;
+using Int8Grid = Grid<Int8Tree>;
+using Int32Grid = Grid<Int32Tree>;
+using Int64Grid = Grid<Int64Tree>;
+using UInt32Grid = Grid<UInt32Tree>;
+using MaskGrid = Grid<MaskTree>;
+using Vec3DGrid = Grid<Vec3DTree>;
+using Vec2IGrid = Grid<Vec2ITree>;
+using Vec3IGrid = Grid<Vec3ITree>;
+using Vec2SGrid = Grid<Vec2STree>;
+using Vec3SGrid = Grid<Vec3STree>;
+using Vec4SGrid = Grid<Vec4STree>;
+using ScalarGrid = FloatGrid;
+using TopologyGrid = MaskGrid;
+using Vec3dGrid = Vec3DGrid;
+using Vec2fGrid = Vec2SGrid;
+using Vec3fGrid = Vec3SGrid;
+using Vec4fGrid = Vec4SGrid;
+using VectorGrid = Vec3fGrid;
+
+}  // namespace OPENVDB_VERSION_NAME
+}  // namespace openvdb
+
+/** \} */

source/blender/blenkernel/BKE_volume.hh

@@ -14,13 +14,14 @@
 #include "BLI_bounds_types.hh"
 #include "BLI_math_vector_types.hh"
 
+#include "BKE_volume_grid_fwd.hh"
+
 struct Depsgraph;
 struct Main;
 struct Object;
 struct ReportList;
 struct Scene;
 struct Volume;
-struct VolumeGrid;
 struct VolumeGridVector;
 
 /* Module */
@@ -69,54 +70,18 @@ bool BKE_volume_is_loaded(const Volume *volume);
 int BKE_volume_num_grids(const Volume *volume);
 const char *BKE_volume_grids_error_msg(const Volume *volume);
 const char *BKE_volume_grids_frame_filepath(const Volume *volume);
-const VolumeGrid *BKE_volume_grid_get_for_read(const Volume *volume, int grid_index);
-VolumeGrid *BKE_volume_grid_get_for_write(Volume *volume, int grid_index);
-const VolumeGrid *BKE_volume_grid_active_get_for_read(const Volume *volume);
+const blender::bke::VolumeGridData *BKE_volume_grid_get(const Volume *volume, int grid_index);
+blender::bke::VolumeGridData *BKE_volume_grid_get_for_write(Volume *volume, int grid_index);
+const blender::bke::VolumeGridData *BKE_volume_grid_active_get_for_read(const Volume *volume);
 /* Tries to find a grid with the given name. Make sure that the volume has been loaded. */
-const VolumeGrid *BKE_volume_grid_find_for_read(const Volume *volume, const char *name);
-VolumeGrid *BKE_volume_grid_find_for_write(Volume *volume, const char *name);
+const blender::bke::VolumeGridData *BKE_volume_grid_find(const Volume *volume, const char *name);
+blender::bke::VolumeGridData *BKE_volume_grid_find_for_write(Volume *volume, const char *name);
 
 /* Tries to set the name of the velocity field. If no such grid exists with the given base name,
  * this will try common post-fixes in order to detect velocity fields split into multiple grids.
  * Return false if neither finding with the base name nor with the post-fixes succeeded. */
 bool BKE_volume_set_velocity_grid_by_name(Volume *volume, const char *base_name);
 
-/* Grid
- *
- * By default only grid metadata is loaded, for access to the tree and voxels
- * BKE_volume_grid_load must be called first. */
-
-enum VolumeGridType : int8_t {
-  VOLUME_GRID_UNKNOWN = 0,
-  VOLUME_GRID_BOOLEAN,
-  VOLUME_GRID_FLOAT,
-  VOLUME_GRID_DOUBLE,
-  VOLUME_GRID_INT,
-  VOLUME_GRID_INT64,
-  VOLUME_GRID_MASK,
-  VOLUME_GRID_VECTOR_FLOAT,
-  VOLUME_GRID_VECTOR_DOUBLE,
-  VOLUME_GRID_VECTOR_INT,
-  VOLUME_GRID_POINTS,
-};
-
-bool BKE_volume_grid_load(const Volume *volume, const VolumeGrid *grid);
-void BKE_volume_grid_unload(const Volume *volume, const VolumeGrid *grid);
-bool BKE_volume_grid_is_loaded(const VolumeGrid *grid);
-
-/* Metadata */
-
-const char *BKE_volume_grid_name(const VolumeGrid *grid);
-VolumeGridType BKE_volume_grid_type(const VolumeGrid *grid);
-int BKE_volume_grid_channels(const VolumeGrid *grid);
-/**
- * Transformation from index space to object space.
- */
-void BKE_volume_grid_transform_matrix(const VolumeGrid *grid, float mat[4][4]);
-void BKE_volume_grid_transform_matrix_set(const Volume *volume,
-                                          VolumeGrid *volume_grid,
-                                          const float mat[4][4]);
-
 /* Volume Editing
  *
  * These are intended for modifiers to use on evaluated data-blocks.
@@ -130,8 +95,13 @@ void BKE_volume_grid_transform_matrix_set(const Volume *volume,
 Volume *BKE_volume_new_for_eval(const Volume *volume_src);
 Volume *BKE_volume_copy_for_eval(const Volume *volume_src);
 
-VolumeGrid *BKE_volume_grid_add(Volume *volume, const char *name, VolumeGridType type);
-void BKE_volume_grid_remove(Volume *volume, VolumeGrid *grid);
+void BKE_volume_grid_remove(Volume *volume, const blender::bke::VolumeGridData *grid);
+
+/**
+ * Adds a new grid to the volume with the name stored in the grid. The caller is responsible for
+ * making sure that the user count already contains the volume as a user.
+ */
+void BKE_volume_grid_add(Volume *volume, const blender::bke::VolumeGridData &grid);
 
 /**
  * OpenVDB crashes when the determinant of the transform matrix becomes too small.
@@ -148,9 +118,4 @@ bool BKE_volume_save(const Volume *volume,
                      ReportList *reports,
                      const char *filepath);
 
-/* OpenVDB Grid Access
- *
- * Access to OpenVDB grid for C++. These will automatically load grids from
- * file or copy shared grids to make them writeable. */
-
 std::optional<blender::Bounds<blender::float3>> BKE_volume_min_max(const Volume *volume);

source/blender/blenkernel/BKE_volume_enums.hh

@@ -0,0 +1,23 @@
+/* SPDX-FileCopyrightText: 2023 Blender Foundation
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#pragma once
+
+/** \file
+ * \ingroup bli
+ */
+
+enum VolumeGridType : int8_t {
+  VOLUME_GRID_UNKNOWN = 0,
+  VOLUME_GRID_BOOLEAN,
+  VOLUME_GRID_FLOAT,
+  VOLUME_GRID_DOUBLE,
+  VOLUME_GRID_INT,
+  VOLUME_GRID_INT64,
+  VOLUME_GRID_MASK,
+  VOLUME_GRID_VECTOR_FLOAT,
+  VOLUME_GRID_VECTOR_DOUBLE,
+  VOLUME_GRID_VECTOR_INT,
+  VOLUME_GRID_POINTS,
+};

source/blender/blenkernel/BKE_volume_grid.hh

@@ -0,0 +1,425 @@
+/* SPDX-FileCopyrightText: 2023 Blender Foundation
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#pragma once
+
+/** \file
+ * \ingroup bke
+ */
+
+#include "BKE_volume_grid_fwd.hh"
+
+#ifdef WITH_OPENVDB
+
+#  include <functional>
+#  include <mutex>
+#  include <optional>
+
+#  include "BKE_volume_enums.hh"
+#  include "BKE_volume_grid_type_traits.hh"
+
+#  include "BLI_implicit_sharing_ptr.hh"
+#  include "BLI_string_ref.hh"
+
+#  include "openvdb_fwd.hh"
+
+namespace blender::bke::volume_grid {
+
+/**
+ * Main volume grid data structure. It wraps an OpenVDB grid and adds some features on top of it.
+ *
+ * A grid contains the following:
+ * - Transform: The mapping between index and object space. It also determines e.g. the voxel size.
+ * - Meta-data: Contains e.g. the name and grid class (fog volume or sdf) and potentially other
+ *   data.
+ * - Tree: This is the heavy data that contains all the voxel values.
+ *
+ * Features:
+ * - Implicit sharing of the #VolumeGridData: This makes it cheap to copy e.g. a #VolumeGrid<T>,
+ *   because it just increases the number of users. An actual copy is only done when the grid is
+ *   modified.
+ * - Implicit sharing of the referenced OpenVDB tree (not grid): The tree is the heavy piece of
+ *   data that contains all the voxel values. Multiple #VolumeGridData can reference the same tree
+ *   with independent meta-data and transforms. The tree is only actually copied when necessary.
+ * - Lazy loading of the entire grid or just the tree: When constructing the #VolumeGridData it is
+ *   possible to provide a callback that lazy-loads the grid when it is first accessed. This is
+ *   especially benefitial when loading grids from a file and it's not clear in the beginning if
+ *   the tree is actually needed. It's also supported to just load the meta-data and transform
+ *   first and to load the tree only when it's used. This allows e.g. transforming or renaming the
+ *   grid without loading the tree.
+ * - Unloading of the tree: It's possible to unload the tree data when it is not in use. This is
+ *   only supported on a shared grid if the tree could be reloaded (e.g. by reading it from a vdb
+ *   file) and if no one is currently accessing the grid data.
+ */
+class VolumeGridData : public ImplicitSharingMixin {
+ private:
+  /**
+   * Empty struct that exists so that it can be used as token in #VolumeTreeAccessToken.
+   */
+  struct AccessToken {
+  };
+
+  /**
+   * A mutex that needs to be locked whenever working with the data members below.
+   */
+  mutable std::mutex mutex_;
+  /**
+   * The actual grid. Depending on the current state, is in one of multiple possible states:
+   * - Empty: When the grid is lazy-loaded and no meta-data is provided.
+   * - Only meta-data and transform: When the grid is lazy-loaded and initial meta-data is
+   *   provided.
+   * - Complete: When the grid is fully loaded. It then contains the meta-data, transform and tree.
+   *
+   * `std::shared_ptr` is used, because some OpenVDB APIs expect the use of those. Unfortunately,
+   * one can not insert and release data from a `shared_ptr`. Therefore, the grid has to be wrapped
+   * by the `shared_ptr` at all times.
+   *
+   * However, this #VolumeGridData is considered to be the only actual owner of the grid. It is
+   * also considered to be the only owner of the meta-data and transform in the grid. It is
+   * possible to share the tree though.
+   */
+  mutable std::shared_ptr<openvdb::GridBase> grid_;
+  /**
+   * Keeps track of whether the tree in `grid_` is current mutable or shared.
+   */
+  mutable const ImplicitSharingInfo *tree_sharing_info_ = nullptr;
+
+  /** The tree stored in the grid is valid. */
+  mutable bool tree_loaded_ = false;
+  /** The transform stored in the grid is valid. */
+  mutable bool transform_loaded_ = false;
+  /** The meta-data stored in the grid is valid. */
+  mutable bool meta_data_loaded_ = false;
+
+  /**
+   * A function that can load the full grid or also just the tree lazily.
+   */
+  std::function<std::shared_ptr<openvdb::GridBase>()> lazy_load_grid_;
+  /**
+   * An error produced while trying to lazily load the grid.
+   */
+  mutable std::string error_message_;
+  /**
+   * A token that allows detecting whether some code is currently accessing the tree (not grid) or
+   * not. If this variable is the only owner of the `shared_ptr`, no one else has access to the
+   * tree. `shared_ptr` is used here because it makes it very easy to manage a user-count without
+   * much boilerplate.
+   */
+  std::shared_ptr<AccessToken> tree_access_token_;
+
+  friend class VolumeTreeAccessToken;
+
+  /** Private default constructor for internal purposes. */
+  VolumeGridData();
+
+ public:
+  /**
+   * Constructs a new volume grid of the given type where all voxels are inactive and the
+   * background value is the default value (generally zero).
+   */
+  explicit VolumeGridData(VolumeGridType grid_type);
+
+  /**
+   * Constructs a new volume grid from the provided OpenVDB grid of which it takes ownership. The
+   * grid must be moved into this constructor and must not be shared currently.
+   */
+  explicit VolumeGridData(std::shared_ptr<openvdb::GridBase> grid);
+
+  /**
+   * Constructs a new volume grid that loads the underlying OpenVDB data lazily.
+   * \param lazy_load_grid: Function that is called when the data is first needed. It returns the
+   *   new grid or may raise an exception. The returned meta-data and transform are ignored if the
+   *   second parameter is provided here.
+   * \param meta_data_and_transform_grid: An initial grid where the tree may be null. This grid
+   *   might come from e.g. #readAllGridMetadata. This allows working with the transform and
+   *   meta-data without actually loading the tree.
+   */
+  explicit VolumeGridData(std::function<std::shared_ptr<openvdb::GridBase>()> lazy_load_grid,
+                          std::shared_ptr<openvdb::GridBase> meta_data_and_transform_grid = {});
+
+  ~VolumeGridData();
+
+  /**
+   * Get an access token for the underlying tree. This is necessary to be able to detect whether
+   * the grid is currently unused so that it can be safely unloaded.
+   */
+  VolumeTreeAccessToken tree_access_token() const;
+
+  /**
+   * Create a copy of the volume grid. This should generally only be done when the current grid is
+   * shared and one owner wants to modify it.
+   *
+   * This makes a deep copy of the transform and meta-data, but the tree remains shared and is only
+   * copied later if necessary.
+   */
+  GVolumeGrid copy() const;
+
+  /**
+   * Get the underlying OpenVDB grid for read-only access. This may load the tree lazily if it's
+   * not loaded already.
+   */
+  const openvdb::GridBase &grid(const VolumeTreeAccessToken &tree_access_token) const;
+  /**
+   * Get the underlying OpenVDB grid for read and write access. This may load the tree lazily if
+   * it's not loaded already. It may also make a copy of the tree if it's currently shared.
+   */
+  openvdb::GridBase &grid_for_write(const VolumeTreeAccessToken &tree_access_token);
+
+  /**
+   * Same as #grid and #grid_for_write but returns the grid as a `shared_ptr` so that it can be
+   * used with APIs that only support grids wrapped into one. This method is not supposed to
+   * actually transfer ownership of the grid.
+   */
+  std::shared_ptr<const openvdb::GridBase> grid_ptr(
+      const VolumeTreeAccessToken &tree_access_token) const;
+  std::shared_ptr<openvdb::GridBase> grid_ptr_for_write(
+      const VolumeTreeAccessToken &tree_access_token);
+
+  /**
+   * Get the name of the grid that's stored in the grid meta-data.
+   */
+  std::string name() const;
+  /**
+   * Replace the name of the grid that's stored in the meta-data.
+   */
+  void set_name(StringRef name);
+
+  /**
+   * Get the transform of the grid for read-only access. This may lazily load the data if it's not
+   * yet available.
+   */
+  const openvdb::math::Transform &transform() const;
+  /**
+   * Get the transform of the grid for read and write access. This may lazily load the data if it's
+   * not yet available.
+   */
+  openvdb::math::Transform &transform_for_write();
+
+  /**
+   * Grid type that's derived from the OpenVDB tree type.
+   */
+  VolumeGridType grid_type() const;
+
+  /**
+   * Same as #grid_type() but does not potentially call the lazy-load function to figure out the
+   * grid type. This can be used e.g. by asserts.
+   */
+  std::optional<VolumeGridType> grid_type_without_load() const;
+
+  /**
+   * Grid class that is stored in the grid's meta data.
+   */
+  openvdb::GridClass grid_class() const;
+
+  /**
+   * True if the grid is fully loaded (including the meta-data, transform and tree).
+   */
+  bool is_loaded() const;
+
+  /**
+   * Non-empty string if there was some error when trying to load the volume.
+   */
+  std::string error_message() const;
+
+  /**
+   * Tree if the tree can be loaded again after it has been unloaded.
+   */
+  bool is_reloadable() const;
+
+  /**
+   * Unloads the tree data if it's reloadable and no one is using it right now.
+   */
+  void unload_tree_if_possible() const;
+
+ private:
+  void ensure_grid_loaded() const;
+  void delete_self();
+};
+
+class VolumeTreeAccessToken {
+ private:
+  std::shared_ptr<VolumeGridData::AccessToken> token_;
+
+  friend VolumeGridData;
+
+ public:
+  /** True if the access token can be used with the given grid. */
+  bool valid_for(const VolumeGridData &grid) const;
+
+  /** Revoke the access token to indicating that the tree is not used anymore. */
+  void reset();
+};
+
+/**
+ * A #GVolumeGrid owns a volume grid. Semantically, each #GVolumeGrid is independent but implicit
+ * sharing is used to avoid unnecessary deep copies.
+ */
+class GVolumeGrid {
+ protected:
+  ImplicitSharingPtr<VolumeGridData> data_;
+
+ public:
+  /**
+   * Constructs a #GVolumeGrid that does not contain any data.
+   */
+  GVolumeGrid() = default;
+  /**
+   * Take (shared) ownership of the given grid data. The caller is responsible for making sure that
+   * the user count includes a user for the newly constructed #GVolumeGrid.
+   */
+  explicit GVolumeGrid(const VolumeGridData *data);
+  /**
+   * Constructs a new volume grid that takes unique ownership of the passed in OpenVDB grid.
+   */
+  explicit GVolumeGrid(std::shared_ptr<openvdb::GridBase> grid);
+  /**
+   * Constructs an empty grid of the given type, where all voxels are inactive and the background
+   * is the default value (generally zero).
+   */
+  explicit GVolumeGrid(VolumeGridType grid_type);
+
+  /**
+   * Get the underlying (potentially shared) volume grid data for read-only access.
+   */
+  const VolumeGridData &get() const;
+
+  /**
+   * Get the underlying volume grid data for read and write access. This may make a copy of the
+   * grid data is shared.
+   */
+  VolumeGridData &get_for_write();
+
+  /**
+   * Move ownership of the underlying grid data to the caller.
+   */
+  const VolumeGridData *release();
+
+  /** Makes it more convenient to retrieve data from the grid. */
+  const VolumeGridData *operator->() const;
+
+  /** True if this contains a grid. */
+  operator bool() const;
+
+  /** Converts to a typed VolumeGrid. This asserts if the type is wrong. */
+  template<typename T> VolumeGrid<T> typed() const;
+};
+
+/**
+ * Same as #GVolumeGrid but makes it easier to work with the grid if the type is known at compile
+ * time.
+ */
+template<typename T> class VolumeGrid : public GVolumeGrid {
+ public:
+  using base_type = T;
+
+  VolumeGrid() = default;
+  explicit VolumeGrid(const VolumeGridData *data);
+  explicit VolumeGrid(std::shared_ptr<OpenvdbGridType<T>> grid);
+
+  /**
+   * Wraps the same methods on #VolumeGridData but casts to the correct OpenVDB type.
+   */
+  const OpenvdbGridType<T> &grid(const VolumeTreeAccessToken &tree_access_token) const;
+  OpenvdbGridType<T> &grid_for_write(const VolumeTreeAccessToken &tree_access_token);
+
+ private:
+  void assert_correct_type() const;
+};
+
+/**
+ * Get the volume grid type based on the tree type in the grid.
+ */
+VolumeGridType get_type(const openvdb::GridBase &grid);
+
+/* -------------------------------------------------------------------- */
+/** \name Inline Methods
+ * \{ */
+
+inline GVolumeGrid::GVolumeGrid(const VolumeGridData *data) : data_(data) {}
+
+inline const VolumeGridData &GVolumeGrid::get() const
+{
+  BLI_assert(*this);
+  return *data_.get();
+}
+
+inline const VolumeGridData *GVolumeGrid::release()
+{
+  return data_.release();
+}
+
+inline GVolumeGrid::operator bool() const
+{
+  return bool(data_);
+}
+
+template<typename T> inline VolumeGrid<T> GVolumeGrid::typed() const
+{
+  if (data_) {
+    data_->add_user();
+  }
+  return VolumeGrid<T>(data_.get());
+}
+
+inline const VolumeGridData *GVolumeGrid::operator->() const
+{
+  BLI_assert(*this);
+  return data_.get();
+}
+
+template<typename T>
+inline VolumeGrid<T>::VolumeGrid(const VolumeGridData *data) : GVolumeGrid(data)
+{
+  this->assert_correct_type();
+}
+
+template<typename T>
+inline VolumeGrid<T>::VolumeGrid(std::shared_ptr<OpenvdbGridType<T>> grid)
+    : GVolumeGrid(std::move(grid))
+{
+  this->assert_correct_type();
+}
+
+template<typename T>
+inline const OpenvdbGridType<T> &VolumeGrid<T>::grid(
+    const VolumeTreeAccessToken &tree_access_token) const
+{
+  return static_cast<const OpenvdbGridType<T> &>(data_->grid(tree_access_token));
+}
+
+template<typename T>
+inline OpenvdbGridType<T> &VolumeGrid<T>::grid_for_write(
+    const VolumeTreeAccessToken &tree_access_token)
+{
+  return static_cast<OpenvdbGridType<T> &>(
+      this->get_for_write().grid_for_write(tree_access_token));
+}
+
+template<typename T> inline void VolumeGrid<T>::assert_correct_type() const
+{
+#  ifndef NDEBUG
+  if (data_) {
+    const VolumeGridType expected_type = VolumeGridTraits<T>::EnumType;
+    if (const std::optional<VolumeGridType> actual_type = data_->grid_type_without_load()) {
+      BLI_assert(expected_type == *actual_type);
+    }
+  }
+#  endif
+}
+
+inline bool VolumeTreeAccessToken::valid_for(const VolumeGridData &grid) const
+{
+  return grid.tree_access_token_ == token_;
+}
+
+inline void VolumeTreeAccessToken::reset()
+{
+  token_.reset();
+}
+
+/** \} */
+
+}  // namespace blender::bke::volume_grid
+
+#endif /* WITH_OPENVDB */

source/blender/blenkernel/BKE_volume_grid_file_cache.hh

@@ -0,0 +1,59 @@
+/* SPDX-FileCopyrightText: 2023 Blender Foundation
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#pragma once
+
+/** \file
+ * \ingroup bke
+ */
+
+#ifdef WITH_OPENVDB
+
+#  include "BLI_vector.hh"
+
+#  include "BKE_volume_grid.hh"
+
+/**
+ * The global volume grid file cache makes it easy to load volumes only once from disk and to then
+ * reuse the loaded volume across Blender. Additionally, this also supports caching simplify
+ * levels which are used when the "volume resolution" simplify scene setting is reduced. Working
+ * with reduced resolution can improve performance and uses less memory.
+ */
+namespace blender::bke::volume_grid::file_cache {
+
+/**
+ * Get the volume grid identified by the parameters from a cache. This does not load the tree data
+ * in grid because that is done on demand when it is accessed.
+ */
+GVolumeGrid get_grid_from_file(StringRef file_path, StringRef grid_name, int simplify_level = 0);
+
+struct GridsFromFile {
+  /**
+   * Empty on success. Otherwise it contains information about why loading the file failed.
+   */
+  std::string error_message;
+  /**
+   * Meta data for the entire file (not for individual grids).
+   */
+  std::shared_ptr<openvdb::MetaMap> file_meta_data;
+  /**
+   * All grids stored in the file.
+   */
+  Vector<GVolumeGrid> grids;
+};
+
+/**
+ * Get all the data stored in a .vdb file. This does not actually load the tree data, which is done
+ * on demand.
+ */
+GridsFromFile get_all_grids_from_file(StringRef file_path, int simplify_level = 0);
+
+/**
+ * Remove all cached volume grids that are currently not referenced outside of the cache.
+ */
+void unload_unused();
+
+}  // namespace blender::bke::volume_grid::file_cache
+
+#endif

source/blender/blenkernel/BKE_volume_grid_fwd.hh

@@ -0,0 +1,119 @@
+/* SPDX-FileCopyrightText: 2023 Blender Foundation
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#pragma once
+
+/** \file
+ * \ingroup bke
+ */
+
+#include <string>
+
+#include "BKE_volume_enums.hh"
+
+#include "BLI_math_matrix_types.hh"
+
+/**
+ * This header gives contains declarations for dealing with volume grids without requiring
+ * including any OpenVDB headers (which can have a significant impact on compile times).
+ *
+ * These functions are available even if `WITH_OPENVDB` is false, but they may just be empty.
+ */
+
+namespace blender::bke::volume_grid {
+
+/**
+ * Wraps an OpenVDB grid and adds features like implicit sharing and lazy-loading.
+ */
+class VolumeGridData;
+
+/**
+ * Owning container for a #VolumeGridData instance.
+ */
+class GVolumeGrid;
+
+/**
+ * Same as #GVolumeGrid but means that the contained grid is of a specific type.
+ */
+template<typename T> class VolumeGrid;
+
+/**
+ * Access token required to use the tree stored in a volume grid. This allows detecting whether a
+ * tree is currently used or not, for the purpose of safely freeing unused trees.
+ */
+class VolumeTreeAccessToken;
+
+/**
+ * Compile time check to see of a type is a #VolumeGrid. This is false for e.g. `float` or
+ * `GVolumeGrid` and true for e.g. `VolumeGrid<int>` and `VolumeGrid<float>`.
+ */
+template<typename T> static constexpr bool is_VolumeGrid_v = false;
+template<typename T> static constexpr bool is_VolumeGrid_v<VolumeGrid<T>> = true;
+
+/**
+ * Get the name stored in the volume grid, e.g. "density".
+ */
+std::string get_name(const VolumeGridData &grid);
+
+/**
+ * Get the data type stored in the volume grid.
+ */
+VolumeGridType get_type(const VolumeGridData &grid);
+
+/**
+ * Get the number of primitive values stored per voxel. For example, for a float-grid this is 1 and
+ * for a vector-grid it is 3 (for x, y and z).
+ */
+int get_channels_num(VolumeGridType type);
+
+/**
+ * Unloads the tree data if no one is using it right now and it could be reloaded later on.
+ */
+void unload_tree_if_possible(const VolumeGridData &grid);
+
+/**
+ * Get the transform of the grid as an affine matrix.
+ */
+float4x4 get_transform_matrix(const VolumeGridData &grid);
+
+/**
+ * Replaces the transform matrix with the given one.
+ */
+void set_transform_matrix(VolumeGridData &grid, const float4x4 &matrix);
+
+/**
+ * Clears the tree data in the grid, but keeps meta-data and the transform intact.
+ */
+void clear_tree(VolumeGridData &grid);
+
+/**
+ * Makes sure that the volume grid is loaded afterwards. This is necessary to call this for
+ * correctness, because the grid will be loaded on demand anyway. Sometimes it may be benefitial
+ * for performance to load the grid eagerly though.
+ */
+void load(const VolumeGridData &grid);
+
+/**
+ * Returns a non-empty string if there was some error when the grid was loaded.
+ */
+std::string error_message_from_load(const VolumeGridData &grid);
+
+/**
+ * True if the full grid (including meta-data, transform and the tree) is already available and
+ * does not have to be loaded lazily anymore.
+ */
+bool is_loaded(const VolumeGridData &grid);
+
+}  // namespace blender::bke::volume_grid
+
+/**
+ * Put the most common types directly into the `blender::bke` namespace.
+ */
+namespace blender::bke {
+using volume_grid::GVolumeGrid;
+using volume_grid::is_VolumeGrid_v;
+using volume_grid::VolumeGrid;
+using volume_grid::VolumeGridData;
+using volume_grid::VolumeTreeAccessToken;
+}  // namespace blender::bke

source/blender/blenkernel/BKE_volume_grid_type_traits.hh

@@ -0,0 +1,119 @@
+/* SPDX-FileCopyrightText: 2023 Blender Foundation
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later */
+
+#pragma once
+
+/** \file
+ * \ingroup bke
+ */
+
+#ifdef WITH_OPENVDB
+
+#  include "BLI_math_quaternion_types.hh"
+#  include "BLI_math_vector_types.hh"
+
+#  include "BKE_volume_enums.hh"
+#  include "openvdb_fwd.hh"
+
+namespace blender::bke {
+
+/**
+ * We uses the native math types from OpenVDB when working with grids. For example, vector grids
+ * contain `openvdb::Vec3f`. #VolumeGridTraits allows mapping between Blender's math types and the
+ * ones from OpenVDB. This allows e.g. using `VolumeGrid<float3>` when the actual grid is a
+ * `openvdb::Vec3SGrid`. The benefit of this is that most places in Blender can keep using our own
+ * types, while only the code that deals with OpenVDB specifically has to care about the mapping
+ * between math type representations.
+ *
+ * \param T The Blender type that we want to get the grid traits for (e.g. `blender::float3`).
+ */
+template<typename T> struct VolumeGridTraits {
+  /**
+   * The type that Blender uses to represent values of the voxel type (e.g. `blender::float3`).
+   */
+  using BlenderType = void;
+  /**
+   * The type that OpenVDB uses.
+   */
+  using PrimitiveType = void;
+  /**
+   * The standard tree type we use for grids of the given type.
+   */
+  using TreeType = void;
+  /**
+   * The corresponding #VolumeGridType for the type.
+   */
+  static constexpr VolumeGridType EnumType = VOLUME_GRID_UNKNOWN;
+};
+
+template<> struct VolumeGridTraits<bool> {
+  using BlenderType = bool;
+  using PrimitiveType = bool;
+  using TreeType = openvdb::BoolTree;
+  static constexpr VolumeGridType EnumType = VOLUME_GRID_BOOLEAN;
+
+  static bool to_openvdb(const bool &value)
+  {
+    return value;
+  }
+  static bool to_blender(const bool &value)
+  {
+    return value;
+  }
+};
+
+template<> struct VolumeGridTraits<int> {
+  using BlenderType = int;
+  using PrimitiveType = int;
+  using TreeType = openvdb::Int32Tree;
+  static constexpr VolumeGridType EnumType = VOLUME_GRID_INT;
+
+  static int to_openvdb(const int &value)
+  {
+    return value;
+  }
+  static int to_blender(const int &value)
+  {
+    return value;
+  }
+};
+
+template<> struct VolumeGridTraits<float> {
+  using BlenderType = float;
+  using PrimitiveType = float;
+  using TreeType = openvdb::FloatTree;
+  static constexpr VolumeGridType EnumType = VOLUME_GRID_FLOAT;
+
+  static float to_openvdb(const float &value)
+  {
+    return value;
+  }
+  static float to_blender(const float &value)
+  {
+    return value;
+  }
+};
+
+template<> struct VolumeGridTraits<float3> {
+  using BlenderType = float3;
+  using PrimitiveType = openvdb::Vec3f;
+  using TreeType = openvdb::Vec3STree;
+  static constexpr VolumeGridType EnumType = VOLUME_GRID_VECTOR_FLOAT;
+
+  static openvdb::Vec3f to_openvdb(const float3 &value)
+  {
+    return openvdb::Vec3f(*value);
+  }
+  static float3 to_blender(const openvdb::Vec3f &value)
+  {
+    return float3(value.asV());
+  }
+};
+
+template<typename T> using OpenvdbTreeType = typename VolumeGridTraits<T>::TreeType;
+template<typename T> using OpenvdbGridType = openvdb::Grid<OpenvdbTreeType<T>>;
+
+}  // namespace blender::bke
+
+#endif /* WITH_OPENVDB */

source/blender/blenkernel/BKE_volume_openvdb.hh

@@ -8,14 +8,20 @@
 
 #  include <openvdb/openvdb.h>
 #  include <openvdb/points/PointDataGrid.h>
+#  include <optional>
 
+#  include "BLI_bounds_types.hh"
 #  include "BLI_math_matrix_types.hh"
 #  include "BLI_math_vector_types.hh"
 #  include "BLI_string_ref.hh"
 
-VolumeGrid *BKE_volume_grid_add_vdb(Volume &volume,
-                                    blender::StringRef name,
-                                    openvdb::GridBase::Ptr vdb_grid);
+#  include "BKE_volume_enums.hh"
+
+struct Volume;
+
+blender::bke::VolumeGridData *BKE_volume_grid_add_vdb(Volume &volume,
+                                                      blender::StringRef name,
+                                                      openvdb::GridBase::Ptr vdb_grid);
 
 std::optional<blender::Bounds<blender::float3>> BKE_volume_grid_bounds(
     openvdb::GridBase::ConstPtr grid);
@@ -28,18 +34,6 @@ std::optional<blender::Bounds<blender::float3>> BKE_volume_grid_bounds(
 openvdb::GridBase::ConstPtr BKE_volume_grid_shallow_transform(openvdb::GridBase::ConstPtr grid,
                                                               const blender::float4x4 &transform);
 
-openvdb::GridBase::ConstPtr BKE_volume_grid_openvdb_for_metadata(const VolumeGrid *grid);
-openvdb::GridBase::ConstPtr BKE_volume_grid_openvdb_for_read(const Volume *volume,
-                                                             const VolumeGrid *grid);
-openvdb::GridBase::Ptr BKE_volume_grid_openvdb_for_write(const Volume *volume,
-                                                         VolumeGrid *grid,
-                                                         bool clear);
-
-void BKE_volume_grid_clear_tree(Volume &volume, VolumeGrid &volume_grid);
-void BKE_volume_grid_clear_tree(openvdb::GridBase &grid);
-
-VolumeGridType BKE_volume_grid_type_openvdb(const openvdb::GridBase &grid);
-
 template<typename OpType>
 auto BKE_volume_grid_type_operation(const VolumeGridType grid_type, OpType &&op)
 {

source/blender/blenkernel/BKE_volume_render.hh

@@ -13,9 +13,10 @@
 
 #include "DNA_volume_types.h"
 
+#include "BKE_volume_enums.hh"
+#include "BKE_volume_grid_fwd.hh"
+
 struct Volume;
-struct VolumeGrid;
-enum VolumeGridType : int8_t;
 
 /* Dense Voxels */
 
@@ -28,7 +29,7 @@ struct DenseFloatVolumeGrid {
 };
 
 bool BKE_volume_grid_dense_floats(const Volume *volume,
-                                  const VolumeGrid *volume_grid,
+                                  const blender::bke::VolumeGridData *volume_grid,
                                   DenseFloatVolumeGrid *r_dense_grid);
 void BKE_volume_dense_float_grid_clear(DenseFloatVolumeGrid *dense_grid);
 
@@ -38,7 +39,7 @@ typedef void (*BKE_volume_wireframe_cb)(
     void *userdata, const float (*verts)[3], const int (*edges)[2], int totvert, int totedge);
 
 void BKE_volume_grid_wireframe(const Volume *volume,
-                               const VolumeGrid *volume_grid,
+                               const blender::bke::VolumeGridData *volume_grid,
                                BKE_volume_wireframe_cb cb,
                                void *cb_userdata);
 
@@ -48,7 +49,7 @@ using BKE_volume_selection_surface_cb =
     void (*)(void *userdata, float (*verts)[3], int (*tris)[3], int totvert, int tottris);
 
 void BKE_volume_grid_selection_surface(const Volume *volume,
-                                       const VolumeGrid *volume_grid,
+                                       const blender::bke::VolumeGridData *volume_grid,
                                        BKE_volume_selection_surface_cb cb,
                                        void *cb_userdata);
 

source/blender/blenkernel/CMakeLists.txt

@@ -309,6 +309,8 @@ set(SRC
   intern/vfontdata_freetype.cc
   intern/viewer_path.cc
   intern/volume.cc
+  intern/volume_grid.cc
+  intern/volume_grid_file_cache.cc
   intern/volume_render.cc
   intern/volume_to_mesh.cc
   intern/workspace.cc
@@ -508,6 +510,9 @@ set(SRC
   BKE_vfontdata.hh
   BKE_viewer_path.hh
   BKE_volume.hh
+  BKE_volume_enums.hh
+  BKE_volume_grid.hh
+  BKE_volume_grid_file_cache.hh
   BKE_volume_openvdb.hh
   BKE_volume_render.hh
   BKE_volume_to_mesh.hh
@@ -853,6 +858,7 @@ if(WITH_GTESTS)
     intern/main_test.cc
     intern/nla_test.cc
     intern/tracking_test.cc
+    intern/volume_test.cc
   )
   set(TEST_INC
     ../editors/include

source/blender/blenkernel/intern/volume.cc

@@ -45,6 +45,8 @@
 #include "BKE_report.h"
 #include "BKE_scene.h"
 #include "BKE_volume.hh"
+#include "BKE_volume_grid.hh"
+#include "BKE_volume_grid_file_cache.hh"
 #include "BKE_volume_openvdb.hh"
 
 #include "BLT_translation.h"
@@ -66,6 +68,7 @@ using blender::float4x4;
 using blender::IndexRange;
 using blender::StringRef;
 using blender::StringRefNull;
+using blender::bke::GVolumeGrid;
 
 #ifdef WITH_OPENVDB
 #  include <atomic>
@@ -77,403 +80,19 @@ using blender::StringRefNull;
 #  include <openvdb/points/PointDataGrid.h>
 #  include <openvdb/tools/GridTransformer.h>
 
-/* Global Volume File Cache
- *
- * Global cache of grids read from VDB files. This is used for sharing grids
- * between multiple volume datablocks with the same filepath, and sharing grids
- * between original and copy-on-write datablocks created by the depsgraph.
- *
- * There are two types of users. Some datablocks only need the grid metadata,
- * example an original datablock volume showing the list of grids in the
- * properties editor. Other datablocks also need the tree and voxel data, for
- * rendering for example. So, depending on the users the grid in the cache may
- * have a tree or not.
- *
- * When the number of users drops to zero, the grid data is immediately deleted.
- *
- * TODO: also add a cache for OpenVDB files rather than individual grids,
- * so getting the list of grids is also cached.
- * TODO: Further, we could cache openvdb::io::File so that loading a grid
- * does not re-open it every time. But then we have to take care not to run
- * out of file descriptors or prevent other applications from writing to it.
- */
-
-static struct VolumeFileCache {
-  /* Cache Entry */
-  struct Entry {
-    Entry(const std::string &filepath, const openvdb::GridBase::Ptr &grid)
-        : filepath(filepath), grid_name(grid->getName()), grid(grid)
-    {
-    }
-