Geometry Nodes: support packing bakes into .blend files #124230

Open
Jacques Lucke wants to merge 53 commits from JacquesLucke/blender:bake-packed into main

When changing the target branch, be careful to rebase the branch in your fork to match. See documentation.
28 changed files with 1236 additions and 140 deletions

View File

@ -10,6 +10,8 @@
#include "BKE_bake_items_paths.hh"
#include "BKE_bake_items_serialize.hh"
#include "DNA_modifier_types.h"
struct NodesModifierData;
struct Main;
struct Object;
@ -35,8 +37,11 @@ enum class CacheStatus {
struct FrameCache {
SubFrame frame;
BakeState state;
/** Used when the baked data is loaded lazily. */
std::optional<std::string> meta_path;
/**
* Used when the baked data is loaded lazily. The meta data either has to be loaded from a file
* or from an in-memory buffer.
*/
std::optional<std::variant<std::string, Span<std::byte>>> meta_data_source;
};
/**
@ -55,8 +60,11 @@ struct NodeBakeCache {
/** All cached frames sorted by frame. */
Vector<std::unique_ptr<FrameCache>> frames;
/** Where to load blobs from disk when loading the baked data lazily. */
/** Loads blob data from memory when the bake is packed. */
std::unique_ptr<MemoryBlobReader> memory_blob_reader;
/** Where to load blobs from disk when loading the baked data lazily from disk. */
std::optional<std::string> blobs_dir;
/** Used to avoid reading blobs multiple times for different frames. */
std::unique_ptr<BlobReadSharing> blob_sharing;
/** Used to avoid checking if a bake exists many times. */
@ -98,6 +106,8 @@ struct ModifierCache {
SimulationNodeCache *get_simulation_node_cache(const int id);
BakeNodeCache *get_bake_node_cache(const int id);
NodeBakeCache *get_node_bake_cache(const int id);
void reset_cache(int id);
};
/**
@ -106,6 +116,9 @@ struct ModifierCache {
*/
void scene_simulation_states_reset(Scene &scene);
std::optional<NodesModifierBakeTarget> get_node_bake_target(const Object &object,
const NodesModifierData &nmd,
int node_id);
std::optional<BakePath> get_node_bake_path(const Main &bmain,
const Object &object,
const NodesModifierData &nmd,
@ -119,10 +132,14 @@ std::optional<std::string> get_modifier_bake_path(const Main &bmain,
const NodesModifierData &nmd);
/**
* Get the directory that contains all baked data for the given modifier by default.
* Get default directory for baking modifier to disk.
*/
std::string get_default_modifier_bake_directory(const Main &bmain,
const Object &object,
const NodesModifierData &nmd);
std::string get_default_node_bake_directory(const Main &bmain,
const Object &object,
const NodesModifierData &nmd,
int node_id);
} // namespace blender::bke::bake

View File

@ -0,0 +1,21 @@
/* SPDX-FileCopyrightText: 2024 Blender Authors
*
* SPDX-License-Identifier: GPL-2.0-or-later */
#pragma once
#include "DNA_modifier_types.h"
#include "BKE_bake_items_paths.hh"
struct ReportList;
namespace blender::bke::bake {
NodesModifierPackedBake *pack_bake_from_disk(const BakePath &bake_path, ReportList *reports);
[[nodiscard]] bool unpack_bake_to_disk(const NodesModifierPackedBake &packed_bake,
const BakePath &bake_path,
ReportList *reports);
} // namespace blender::bke::bake

View File

@ -36,7 +36,7 @@ struct BakePath {
};
std::string frame_to_file_name(const SubFrame &frame);
std::optional<SubFrame> file_name_to_frame(const StringRefNull file_name);
std::optional<SubFrame> file_name_to_frame(const StringRef file_name);
Vector<MetaFile> find_sorted_meta_files(const StringRefNull meta_dir);

View File

@ -47,6 +47,9 @@ class BlobReader {
* Abstract base class for writing binary data.
*/
class BlobWriter {
protected:
int64_t total_written_size_ = 0;
public:
/**
* Write the provided binary data.
@ -62,6 +65,11 @@ class BlobWriter {
*/
virtual BlobSlice write_as_stream(StringRef file_extension,
FunctionRef<void(std::ostream &)> fn);
int64_t written_size() const
{
return total_written_size_;
}
};
/**
@ -184,6 +192,49 @@ class DiskBlobWriter : public BlobWriter {
FunctionRef<void(std::ostream &)> fn) override;
};
/**
* A specific #BlobWriter that keeps all data in memory.
*/
class MemoryBlobWriter : public BlobWriter {
public:
struct OutputStream {
std::unique_ptr<std::ostringstream> stream;
int64_t offset = 0;
};
private:
std::string base_name_;
std::string blob_name_;
Map<std::string, OutputStream> stream_by_name_;
int independent_file_count_ = 0;
public:
MemoryBlobWriter(std::string base_name);
BlobSlice write(const void *data, int64_t size) override;
BlobSlice write_as_stream(StringRef file_extension,
FunctionRef<void(std::ostream &)> fn) override;
const Map<std::string, OutputStream> &get_stream_by_name() const
{
return stream_by_name_;
}
};
/**
* A specific #BlobReader that reads data from in-memory buffers.
*/
class MemoryBlobReader : public BlobReader {
private:
Map<StringRef, Span<std::byte>> blob_by_name_;
public:
void add(StringRef name, Span<std::byte> blob);
[[nodiscard]] bool read(const BlobSlice &slice, void *r_data) const override;
};
void serialize_bake(const BakeState &bake_state,
BlobWriter &blob_writer,
BlobWriteSharing &blob_sharing,

View File

@ -31,7 +31,7 @@ extern "C" {
/* Blender file format version. */
#define BLENDER_FILE_VERSION BLENDER_VERSION
#define BLENDER_FILE_SUBVERSION 22
#define BLENDER_FILE_SUBVERSION 23
/* Minimum Blender version that supports reading file written with the current
* version. Older Blender versions will test this and cancel loading the file, showing a warning to

View File

@ -108,7 +108,20 @@ void BKE_packedfile_free(PackedFile *pf);
/* Info. */
int BKE_packedfile_count_all(Main *bmain);
struct PackedFileCount {
/** Counts e.g. packed images and sounds. */
int individual_files = 0;
/** Counts bakes that may consist of multiple files. */
int bakes = 0;
int total() const
{
return this->individual_files + this->bakes;
}
};
PackedFileCount BKE_packedfile_count_all(Main *bmain);
/**
* This function compares a packed file to a 'real' file.
* It returns an integer indicating if:

View File

@ -69,6 +69,7 @@ set(SRC
intern/autoexec.cc
intern/bake_data_block_map.cc
intern/bake_geometry_nodes_modifier.cc
intern/bake_geometry_nodes_modifier_pack.cc
intern/bake_items.cc
intern/bake_items_paths.cc
intern/bake_items_serialize.cc
@ -340,6 +341,7 @@ set(SRC
BKE_bake_data_block_id.hh
BKE_bake_data_block_map.hh
BKE_bake_geometry_nodes_modifier.hh
BKE_bake_geometry_nodes_modifier_pack.hh
BKE_bake_items.hh
BKE_bake_items_paths.hh
BKE_bake_items_serialize.hh

View File

@ -71,6 +71,16 @@ NodeBakeCache *ModifierCache::get_node_bake_cache(const int id)
return nullptr;
}
void ModifierCache::reset_cache(const int id)
{
if (SimulationNodeCache *cache = this->get_simulation_node_cache(id)) {
cache->reset();
}
if (BakeNodeCache *cache = this->get_bake_node_cache(id)) {
cache->reset();
}
}
void scene_simulation_states_reset(Scene &scene)
{
FOREACH_SCENE_OBJECT_BEGIN (&scene, ob) {
@ -107,6 +117,23 @@ std::optional<std::string> get_modifier_bake_path(const Main &bmain,
return absolute_bake_dir;
}
std::optional<NodesModifierBakeTarget> get_node_bake_target(const Object & /*object*/,
const NodesModifierData &nmd,
int node_id)
{
const NodesModifierBake *bake = nmd.find_bake(node_id);
if (!bake) {
return std::nullopt;
}
if (bake->bake_target != NODES_MODIFIER_BAKE_TARGET_INHERIT) {
return NodesModifierBakeTarget(bake->bake_target);
}
if (nmd.bake_target != NODES_MODIFIER_BAKE_TARGET_INHERIT) {
return NodesModifierBakeTarget(nmd.bake_target);
}
return NODES_MODIFIER_BAKE_TARGET_PACKED;
}
std::optional<bake::BakePath> get_node_bake_path(const Main &bmain,
const Object &object,
const NodesModifierData &nmd,
@ -217,4 +244,19 @@ std::string get_default_modifier_bake_directory(const Main &bmain,
return dir;
}
std::string get_default_node_bake_directory(const Main &bmain,
const Object &object,
const NodesModifierData &nmd,
int node_id)
{
char dir[FILE_MAX];
BLI_path_join(dir,
sizeof(dir),
"//",
get_blend_file_name(bmain).c_str(),
get_modifier_directory_name(object, nmd.modifier).c_str(),
std::to_string(node_id).c_str());
return dir;
}
} // namespace blender::bke::bake

View File

@ -0,0 +1,106 @@
/* SPDX-FileCopyrightText: 2024 Blender Authors
*
* SPDX-License-Identifier: GPL-2.0-or-later */
#include "BKE_bake_geometry_nodes_modifier_pack.hh"
#include "BKE_packedFile.hh"
#include "BKE_report.hh"
#include "BLI_fileops.hh"
#include "BLI_path_util.h"
#include "BLI_string.h"
namespace blender::bke::bake {
static Vector<NodesModifierBakeFile> pack_files_from_directory(const StringRefNull directory,
ReportList *reports)
{
if (!BLI_is_dir(directory.c_str())) {
BKE_reportf(reports, RPT_ERROR, "%s is no directory", directory.c_str());
return {};
}
direntry *dir_entries = nullptr;
const int dir_entries_num = BLI_filelist_dir_contents(directory.c_str(), &dir_entries);
BLI_SCOPED_DEFER([&]() { BLI_filelist_free(dir_entries, dir_entries_num); });
Vector<NodesModifierBakeFile> bake_files;
for (const int i : IndexRange(dir_entries_num)) {
const direntry &dir_entry = dir_entries[i];
const StringRefNull dir_entry_path = dir_entry.path;
const StringRefNull name = dir_entry.relname;
NodesModifierBakeFile bake_file;
bake_file.name = BLI_strdup_null(name.c_str());
bake_file.packed_file = BKE_packedfile_new(reports, dir_entry_path.c_str(), "");
if (bake_file.packed_file) {
bake_files.append(bake_file);
}
}
return bake_files;
}
NodesModifierPackedBake *pack_bake_from_disk(const BakePath &bake_path, ReportList *reports)
{
const Vector<NodesModifierBakeFile> meta_bake_files = pack_files_from_directory(
bake_path.meta_dir, reports);
if (meta_bake_files.is_empty()) {
return nullptr;
}
const Vector<NodesModifierBakeFile> blob_bake_files = pack_files_from_directory(
bake_path.blobs_dir, reports);
NodesModifierPackedBake *packed_bake = MEM_cnew<NodesModifierPackedBake>(__func__);
packed_bake->meta_files_num = meta_bake_files.size();
packed_bake->blob_files_num = blob_bake_files.size();
packed_bake->meta_files = MEM_cnew_array<NodesModifierBakeFile>(packed_bake->meta_files_num,
__func__);
packed_bake->blob_files = MEM_cnew_array<NodesModifierBakeFile>(packed_bake->blob_files_num,
__func__);
uninitialized_copy_n(meta_bake_files.data(), meta_bake_files.size(), packed_bake->meta_files);
uninitialized_copy_n(blob_bake_files.data(), blob_bake_files.size(), packed_bake->blob_files);
return packed_bake;
}
bool unpack_bake_to_disk(const NodesModifierPackedBake &packed_bake,
const BakePath &bake_path,
ReportList *reports)
{
auto unpack_file = [&](const StringRefNull directory, const NodesModifierBakeFile &bake_file) {
char file_path[FILE_MAX];
BLI_path_join(file_path, sizeof(file_path), directory.c_str(), bake_file.name);
if (!BLI_file_ensure_parent_dir_exists(file_path)) {
BKE_reportf(reports, RPT_ERROR, "Can't ensure directory: %s", directory.c_str());
return false;
}
fstream fs(file_path, std::ios::out);
fs.write(static_cast<const char *>(bake_file.packed_file->data), bake_file.packed_file->size);
if (fs.bad()) {
BKE_reportf(reports, RPT_ERROR, "Can't write file : %s", file_path);
return false;
}
return true;
};
for (const NodesModifierBakeFile &bake_file :
Span{packed_bake.meta_files, packed_bake.meta_files_num})
{
if (!unpack_file(bake_path.meta_dir, bake_file)) {
return false;
}
}
for (const NodesModifierBakeFile &bake_file :
Span{packed_bake.blob_files, packed_bake.blob_files_num})
{
if (!unpack_file(bake_path.blobs_dir, bake_file)) {
return false;
}
}
return true;
}
} // namespace blender::bke::bake

View File

@ -19,13 +19,18 @@ std::string frame_to_file_name(const SubFrame &frame)
return file_name_c;
}
std::optional<SubFrame> file_name_to_frame(const StringRefNull file_name)
std::optional<SubFrame> file_name_to_frame(const StringRef file_name)
{
char modified_file_name[FILE_MAX];
STRNCPY(modified_file_name, file_name.c_str());
file_name.copy(modified_file_name);
BLI_string_replace_char(modified_file_name, '_', '.');
const SubFrame frame = std::stof(modified_file_name);
return frame;
try {
const SubFrame frame = std::stof(modified_file_name);
return frame;
}
catch (...) {
return std::nullopt;
}
}
Vector<MetaFile> find_sorted_meta_files(const StringRefNull meta_dir)

View File

@ -19,6 +19,7 @@
#include "BLI_path_util.h"
#include "DNA_material_types.h"
#include "DNA_modifier_types.h"
#include "DNA_volume_types.h"
#include "RNA_access.hh"
@ -126,16 +127,24 @@ BlobSlice DiskBlobWriter::write(const void *data, const int64_t size)
const int64_t old_offset = current_offset_;
blob_stream_.write(static_cast<const char *>(data), size);
current_offset_ += size;
total_written_size_ += size;
return {blob_name_, {old_offset, size}};
}
static std::string make_independent_file_name(const StringRef base_name,
const int file_index,
const StringRef extension)
{
return fmt::format("{}_file_{}{}", base_name, file_index, extension);
}
BlobSlice DiskBlobWriter::write_as_stream(const StringRef file_extension,
const FunctionRef<void(std::ostream &)> fn)
{
BLI_assert(file_extension.startswith("."));
independent_file_count_++;
const std::string file_name = fmt::format(
"{}_file_{}{}", base_name_, independent_file_count_, file_extension);
const std::string file_name = make_independent_file_name(
base_name_, independent_file_count_, file_extension);
char path[FILE_MAX];
BLI_path_join(path, sizeof(path), blob_dir_.c_str(), file_name.c_str());
@ -143,9 +152,60 @@ BlobSlice DiskBlobWriter::write_as_stream(const StringRef file_extension,
std::fstream stream{path, std::ios::out | std::ios::binary};
fn(stream);
const int64_t written_bytes_num = stream.tellg();
total_written_size_ += written_bytes_num;
return {file_name, {0, written_bytes_num}};
}
void MemoryBlobReader::add(const StringRef name, const Span<std::byte> blob)
{
blob_by_name_.add(name, blob);
}
bool MemoryBlobReader::read(const BlobSlice &slice, void *r_data) const
{
if (slice.range.is_empty()) {
return true;
}
const Span<std::byte> blob_data = blob_by_name_.lookup_default(slice.name, {});
if (!blob_data.index_range().contains(slice.range)) {
return false;
}
const void *copy_src = blob_data.slice(slice.range).data();
memcpy(r_data, copy_src, slice.range.size());
return true;
}
MemoryBlobWriter::MemoryBlobWriter(std::string base_name) : base_name_(std::move(base_name))
{
blob_name_ = base_name_ + ".blob";
stream_by_name_.add(blob_name_, {std::make_unique<std::ostringstream>(std::ios::binary)});
}
BlobSlice MemoryBlobWriter::write(const void *data, int64_t size)
{
OutputStream &stream = stream_by_name_.lookup(blob_name_);
const int64_t old_offset = stream.offset;
stream.stream->write(static_cast<const char *>(data), size);
stream.offset += size;
total_written_size_ += size;
return {blob_name_, IndexRange::from_begin_size(old_offset, size)};
}
BlobSlice MemoryBlobWriter::write_as_stream(const StringRef file_extension,
const FunctionRef<void(std::ostream &)> fn)
{
BLI_assert(file_extension.startswith("."));
independent_file_count_++;
const std::string name = make_independent_file_name(
base_name_, independent_file_count_, file_extension);
OutputStream stream{std::make_unique<std::ostringstream>(std::ios::binary)};
fn(*stream.stream);
const int64_t size = stream.stream->tellp();
stream_by_name_.add_new(name, std::move(stream));
total_written_size_ += size;
return {base_name_, IndexRange(size)};
}
BlobWriteSharing::~BlobWriteSharing()
{
for (const ImplicitSharingInfo *sharing_info : stored_by_runtime_.keys()) {

View File

@ -20,6 +20,7 @@
#include "DNA_ID.h"
#include "DNA_image_types.h"
#include "DNA_modifier_types.h"
#include "DNA_packedFile_types.h"
#include "DNA_sound_types.h"
#include "DNA_vfont_types.h"
@ -28,6 +29,8 @@
#include "BLI_blenlib.h"
#include "BLI_utildefines.h"
#include "BKE_bake_geometry_nodes_modifier.hh"
#include "BKE_bake_geometry_nodes_modifier_pack.hh"
#include "BKE_image.h"
#include "BKE_image_format.h"
#include "BKE_main.hh"
@ -37,9 +40,13 @@
#include "BKE_vfont.hh"
#include "BKE_volume.hh"
#include "DEG_depsgraph.hh"
#include "IMB_imbuf.hh"
#include "IMB_imbuf_types.hh"
#include "MOD_nodes.hh"
#include "BLO_read_write.hh"
#include "CLG_log.h"
@ -108,26 +115,27 @@ int BKE_packedfile_read(PackedFile *pf, void *data, int size)
return size;
}
int BKE_packedfile_count_all(Main *bmain)
PackedFileCount BKE_packedfile_count_all(Main *bmain)
{
Image *ima;
VFont *vf;
bSound *sound;
Volume *volume;
int count = 0;
PackedFileCount count;
/* let's check if there are packed files... */
for (ima = static_cast<Image *>(bmain->images.first); ima;
ima = static_cast<Image *>(ima->id.next))
{
if (BKE_image_has_packedfile(ima) && !ID_IS_LINKED(ima)) {
count++;
count.individual_files++;
}
}
for (vf = static_cast<VFont *>(bmain->fonts.first); vf; vf = static_cast<VFont *>(vf->id.next)) {
if (vf->packedfile && !ID_IS_LINKED(vf)) {
count++;
count.individual_files++;
}
}
@ -135,7 +143,7 @@ int BKE_packedfile_count_all(Main *bmain)
sound = static_cast<bSound *>(sound->id.next))
{
if (sound->packedfile && !ID_IS_LINKED(sound)) {
count++;
count.individual_files++;
}
}
@ -143,7 +151,23 @@ int BKE_packedfile_count_all(Main *bmain)
volume = static_cast<Volume *>(volume->id.next))
{
if (volume->packedfile && !ID_IS_LINKED(volume)) {
count++;
count.individual_files++;
}
}
LISTBASE_FOREACH (Object *, object, &bmain->objects) {
if (ID_IS_LINKED(object)) {
continue;
}
LISTBASE_FOREACH (ModifierData *, md, &object->modifiers) {
if (md->type == eModifierType_Nodes) {
NodesModifierData *nmd = reinterpret_cast<NodesModifierData *>(md);
for (const NodesModifierBake &bake : blender::Span{nmd->bakes, nmd->bakes_num}) {
if (bake.packed) {
count.bakes++;
}
}
}
}
}
@ -242,6 +266,25 @@ PackedFile *BKE_packedfile_new(ReportList *reports, const char *filepath_rel, co
return pf;
}
static int BKE_packedfile_pack_geometry_nodes_bake(Main &bmain,
ReportList *reports,
Object &object,
NodesModifierData &nmd,
NodesModifierBake &bake)
{
using namespace blender::bke;
const std::optional<bake::BakePath> bake_path = bake::get_node_bake_path(
bmain, object, nmd, bake.id);
if (!bake_path) {
/* Has no data to pack. */
return RET_OK;
}
bake.packed = bake::pack_bake_from_disk(*bake_path, reports);
nmd.runtime->cache->reset_cache(bake.id);
DEG_id_tag_update(&object.id, ID_RECALC_GEOMETRY);
return RET_OK;
}
void BKE_packedfile_pack_all(Main *bmain, ReportList *reports, bool verbose)
{
Image *ima;
@ -299,6 +342,22 @@ void BKE_packedfile_pack_all(Main *bmain, ReportList *reports, bool verbose)
}
}
LISTBASE_FOREACH (Object *, object, &bmain->objects) {
if (ID_IS_LINKED(object)) {
continue;
}
LISTBASE_FOREACH (ModifierData *, md, &object->modifiers) {
if (md->type == eModifierType_Nodes) {
NodesModifierData *nmd = reinterpret_cast<NodesModifierData *>(md);
for (NodesModifierBake &bake : blender::MutableSpan{nmd->bakes, nmd->bakes_num}) {
if (!bake.packed) {
BKE_packedfile_pack_geometry_nodes_bake(*bmain, reports, *object, *nmd, bake);
}
}
}
}
}
if (tot > 0) {
BKE_reportf(reports, RPT_INFO, "Packed %d file(s)", tot);
}
@ -720,6 +779,121 @@ int BKE_packedfile_unpack_volume(Main *bmain,
return ret_value;
}
static bool directory_is_empty(const blender::StringRefNull path)
{
direntry *entries = nullptr;
const int entries_num = BLI_filelist_dir_contents(path.c_str(), &entries);
BLI_filelist_free(entries, entries_num);
return entries_num == 0;
}
static bool disk_bake_exists(const blender::bke::bake::BakePath &path)
{
return !directory_is_empty(path.meta_dir);
}
static int BKE_packedfile_unpack_geometry_nodes_bake(Main &bmain,
ReportList *reports,
Object &object,
NodesModifierData &nmd,
NodesModifierBake &bake,
enum ePF_FileStatus how)
{
using namespace blender;
using namespace blender::bke;
if (StringRef(BKE_main_blendfile_path(&bmain)).is_empty()) {
BKE_report(reports, RPT_ERROR, "Can only unpack bake if the current .blend file is saved");
return RET_ERROR;
}
DEG_id_tag_update(&object.id, ID_RECALC_GEOMETRY);
auto prepare_local_path = [&]() {
const std::string directory = bake::get_default_node_bake_directory(
bmain, object, nmd, bake.id);
bake.flag |= NODES_MODIFIER_BAKE_CUSTOM_PATH;
MEM_SAFE_FREE(bake.directory);
bake.directory = BLI_strdup(directory.c_str());
const char *base_path = ID_BLEND_PATH(&bmain, &object.id);
char absolute_dir[FILE_MAX];
STRNCPY(absolute_dir, directory.c_str());
BLI_path_abs(absolute_dir, base_path);
return bake::BakePath::from_single_root(absolute_dir);
};
auto prepare_original_path = [&]() {
if (const std::optional<bake::BakePath> bake_path = bake::get_node_bake_path(
bmain, object, nmd, bake.id))
{
return *bake_path;
}
return prepare_local_path();
};
auto delete_bake_on_disk = [&](const bake::BakePath &bake_path) {
BLI_delete(bake_path.meta_dir.c_str(), true, true);
BLI_delete(bake_path.blobs_dir.c_str(), true, true);
};
auto free_packed_bake = [&]() {
blender::nodes_modifier_packed_bake_free(bake.packed);
bake.packed = nullptr;
nmd.runtime->cache->reset_cache(bake.id);
};
switch (how) {
case PF_USE_ORIGINAL: {
const bake::BakePath bake_path = prepare_original_path();
if (!disk_bake_exists(bake_path)) {
delete_bake_on_disk(bake_path);
if (!bake::unpack_bake_to_disk(*bake.packed, bake_path, reports)) {
return RET_ERROR;
}
}
free_packed_bake();
return RET_OK;
}
case PF_WRITE_ORIGINAL: {
const bake::BakePath bake_path = prepare_original_path();
delete_bake_on_disk(bake_path);
if (!bake::unpack_bake_to_disk(*bake.packed, bake_path, reports)) {
return RET_ERROR;
}
free_packed_bake();
return RET_OK;
}
case PF_USE_LOCAL: {
const bake::BakePath bake_path = prepare_local_path();
if (!disk_bake_exists(bake_path)) {
delete_bake_on_disk(bake_path);
if (!bake::unpack_bake_to_disk(*bake.packed, bake_path, reports)) {
return RET_ERROR;
}
}
free_packed_bake();
return RET_OK;
}
case PF_WRITE_LOCAL: {
const bake::BakePath bake_path = prepare_local_path();
delete_bake_on_disk(bake_path);
if (!bake::unpack_bake_to_disk(*bake.packed, bake_path, reports)) {
return RET_ERROR;
}
free_packed_bake();
return RET_OK;
}
case PF_KEEP: {
return RET_OK;
}
case PF_REMOVE: {
free_packed_bake();
return RET_OK;
}
default: {
break;
}
}
return RET_ERROR;
}
int BKE_packedfile_unpack_all_libraries(Main *bmain, ReportList *reports)
{
Library *lib;
@ -816,6 +990,22 @@ void BKE_packedfile_unpack_all(Main *bmain, ReportList *reports, enum ePF_FileSt
BKE_packedfile_unpack_volume(bmain, reports, volume, how);
}
}
LISTBASE_FOREACH (Object *, object, &bmain->objects) {
if (ID_IS_LINKED(object)) {
continue;
}
LISTBASE_FOREACH (ModifierData *, md, &object->modifiers) {
if (md->type == eModifierType_Nodes) {
NodesModifierData *nmd = reinterpret_cast<NodesModifierData *>(md);
for (NodesModifierBake &bake : blender::MutableSpan{nmd->bakes, nmd->bakes_num}) {
if (bake.packed) {
BKE_packedfile_unpack_geometry_nodes_bake(*bmain, reports, *object, *nmd, bake, how);
}
}
}
}
}
}
bool BKE_packedfile_id_check(const ID *id)

View File

@ -249,6 +249,20 @@ class IndexRange {
return value >= start_ && value < start_ + size_;
}
constexpr bool contains(const IndexRange range) const
{
if (range.is_empty()) {
return true;
}
if (range.start_ < start_) {
return false;
}
if (range.start_ + range.size_ > start_ + size_) {
return false;
}
return true;
}
/**
* Returns a new range, that contains a sub-interval of the current one.
*/

View File

@ -7,6 +7,7 @@
#include <cmath>
#include "BLI_assert.h"
#include "BLI_hash.hh"
#include "BLI_math_base.h"
#include "BLI_struct_equality_utils.hh"
@ -59,6 +60,11 @@ struct SubFrame {
return {INT32_MAX, std::nexttowardf(1.0f, 0.0)};
}
uint64_t hash() const
{
return get_default_hash(frame_, subframe_);
}
BLI_STRUCT_EQUALITY_OPERATORS_2(SubFrame, frame_, subframe_)
friend bool operator<(const SubFrame &a, const SubFrame &b)

View File

@ -4625,6 +4625,21 @@ void blo_do_versions_400(FileData *fd, Library * /*lib*/, Main *bmain)
add_bevel_modifier_attribute_name_defaults(*bmain);
}
if (!MAIN_VERSION_FILE_ATLEAST(bmain, 403, 23)) {
LISTBASE_FOREACH (Object *, object, &bmain->objects) {
LISTBASE_FOREACH (ModifierData *, md, &object->modifiers) {
if (md->type != eModifierType_Nodes) {
continue;
}
NodesModifierData &nmd = *reinterpret_cast<NodesModifierData *>(md);
if (nmd.bake_target == NODES_MODIFIER_BAKE_TARGET_INHERIT) {
/* Use disk target for existing modifiers to avoid changing behavior. */
nmd.bake_target = NODES_MODIFIER_BAKE_TARGET_DISK;
}
}
}
}
/**
* Always bump subversion in BKE_blender_version.h when adding versioning
* code here, and wrap it inside a MAIN_VERSION_FILE_ATLEAST check.

View File

@ -25,12 +25,14 @@
#include "DNA_windowmanager_types.h"
#include "BKE_bake_geometry_nodes_modifier.hh"
#include "BKE_bake_geometry_nodes_modifier_pack.hh"
#include "BKE_context.hh"
#include "BKE_global.hh"
#include "BKE_lib_id.hh"
#include "BKE_main.hh"
#include "BKE_modifier.hh"
#include "BKE_node_runtime.hh"
#include "BKE_packedFile.hh"
#include "BKE_report.hh"
#include "BKE_scene.hh"
@ -196,12 +198,6 @@ static bool bake_simulation_poll(bContext *C)
if (!ED_operator_object_active(C)) {
return false;
}
Main *bmain = CTX_data_main(C);
const StringRefNull path = BKE_main_blendfile_path(bmain);
if (path.is_empty()) {
CTX_wm_operator_poll_msg_set(C, "File must be saved before baking");
return false;
}
Object *ob = context_active_object(C);
const bool use_frame_cache = ob->flag & OB_FLAG_USE_SIMULATION_CACHE;
if (!use_frame_cache) {
@ -217,7 +213,8 @@ struct NodeBakeRequest {
int bake_id;
int node_type;
bake::BakePath path;
/** Store bake in this location if available, otherwise pack the baked data. */
std::optional<bake::BakePath> path;
int frame_start;
int frame_end;
std::unique_ptr<bake::BlobWriteSharing> blob_sharing;
@ -272,6 +269,19 @@ static void bake_geometry_nodes_startjob(void *customdata, wmJobWorkerStatus *wo
const float progress_per_frame = frame_step_size / frames_to_bake;
const int old_frame = job.scene->r.cfra;
struct MemoryBakeFile {
std::string name;
std::string data;
};
struct PackedBake {
Vector<MemoryBakeFile> meta_files;
Vector<MemoryBakeFile> blob_files;
};
Map<NodeBakeRequest *, PackedBake> packed_data_by_bake;
Map<NodeBakeRequest *, int64_t> size_by_bake;
for (float frame_f = global_bake_start_frame; frame_f <= global_bake_end_frame;
frame_f += frame_step_size)
{
@ -307,23 +317,100 @@ static void bake_geometry_nodes_startjob(void *customdata, wmJobWorkerStatus *wo
continue;
}
const bake::BakePath path = request.path;
int64_t &written_size = size_by_bake.lookup_or_add(&request, 0);
char meta_path[FILE_MAX];
BLI_path_join(meta_path,
sizeof(meta_path),
path.meta_dir.c_str(),
(frame_file_name + ".json").c_str());
BLI_file_ensure_parent_dir_exists(meta_path);
bake::DiskBlobWriter blob_writer{path.blobs_dir, frame_file_name};
fstream meta_file{meta_path, std::ios::out};
bake::serialize_bake(frame_cache.state, blob_writer, *request.blob_sharing, meta_file);
if (request.path.has_value()) {
char meta_path[FILE_MAX];
BLI_path_join(meta_path,
sizeof(meta_path),
request.path->meta_dir.c_str(),
(frame_file_name + ".json").c_str());
BLI_file_ensure_parent_dir_exists(meta_path);
bake::DiskBlobWriter blob_writer{request.path->blobs_dir, frame_file_name};
fstream meta_file{meta_path, std::ios::out};
bake::serialize_bake(frame_cache.state, blob_writer, *request.blob_sharing, meta_file);
written_size += blob_writer.written_size();
written_size += meta_file.tellp();
}
else {
PackedBake &packed_data = packed_data_by_bake.lookup_or_add_default(&request);
bake::MemoryBlobWriter blob_writer{frame_file_name};
std::ostringstream meta_file{std::ios::binary};
bake::serialize_bake(frame_cache.state, blob_writer, *request.blob_sharing, meta_file);
packed_data.meta_files.append({frame_file_name + ".json", meta_file.str()});
const Map<std::string, bake::MemoryBlobWriter::OutputStream> &blob_stream_by_name =
blob_writer.get_stream_by_name();
for (auto &&item : blob_stream_by_name.items()) {
std::string data = item.value.stream->str();
if (data.empty()) {
continue;
}
packed_data.blob_files.append({item.key, std::move(data)});
}
written_size += blob_writer.written_size();
written_size += meta_file.tellp();
}
}
worker_status->progress += progress_per_frame;
worker_status->do_update = true;
}
/* Update bake sizes. */
for (NodeBakeRequest &request : job.bake_requests) {
NodesModifierBake *bake = request.nmd->find_bake(request.bake_id);
bake->bake_size = size_by_bake.lookup_default(&request, 0);
}
/* Store gathered data as packed data. */
for (NodeBakeRequest &request : job.bake_requests) {
NodesModifierBake *bake = request.nmd->find_bake(request.bake_id);
PackedBake *packed_data = packed_data_by_bake.lookup_ptr(&request);
if (!packed_data) {
continue;
}
NodesModifierPackedBake *packed_bake = MEM_cnew<NodesModifierPackedBake>(__func__);
packed_bake->meta_files_num = packed_data->meta_files.size();
packed_bake->blob_files_num = packed_data->blob_files.size();
packed_bake->meta_files = MEM_cnew_array<NodesModifierBakeFile>(packed_bake->meta_files_num,
__func__);
packed_bake->blob_files = MEM_cnew_array<NodesModifierBakeFile>(packed_bake->blob_files_num,
__func__);
auto transfer_to_bake =
[&](NodesModifierBakeFile *bake_files, MemoryBakeFile *memory_bake_files, const int num) {
for (const int i : IndexRange(num)) {
NodesModifierBakeFile &bake_file = bake_files[i];
MemoryBakeFile &memory = memory_bake_files[i];
bake_file.name = BLI_strdup_null(memory.name.c_str());
const int64_t data_size = memory.data.size();
if (data_size == 0) {
continue;
}
void *data = MEM_mallocN(data_size, __func__);
memcpy(data, memory.data.data(), data_size);
memory.data.clear();
memory.data.shrink_to_fit();
bake_file.packed_file = BKE_packedfile_new_from_memory(data, data_size);
}
};
transfer_to_bake(
packed_bake->meta_files, packed_data->meta_files.data(), packed_bake->meta_files_num);
transfer_to_bake(
packed_bake->blob_files, packed_data->blob_files.data(), packed_bake->blob_files_num);
/* Should have been freed before. */
BLI_assert(bake->packed == nullptr);
bake->packed = packed_bake;
}
/* Tag simulations as being baked. */
for (NodeBakeRequest &request : job.bake_requests) {
if (request.node_type != GEO_NODE_SIMULATION_OUTPUT) {
@ -411,6 +498,11 @@ static void try_delete_bake(
}
clear_data_block_references(*bake);
if (bake->packed) {
nodes_modifier_packed_bake_free(bake->packed);
bake->packed = nullptr;
}
const std::optional<bake::BakePath> bake_path = bake::get_node_bake_path(
*bmain, object, nmd, bake_id);
if (!bake_path) {
@ -534,16 +626,14 @@ static Vector<NodeBakeRequest> collect_simulations_to_bake(Main &bmain,
request.bake_id = id;
request.node_type = node->type;
request.blob_sharing = std::make_unique<bake::BlobWriteSharing>();
std::optional<bake::BakePath> path = bake::get_node_bake_path(bmain, *object, *nmd, id);
if (!path) {
continue;
if (bake::get_node_bake_target(*object, *nmd, id) == NODES_MODIFIER_BAKE_TARGET_DISK) {
request.path = bake::get_node_bake_path(bmain, *object, *nmd, id);
}
std::optional<IndexRange> frame_range = bake::get_node_bake_frame_range(
scene, *object, *nmd, id);
if (!frame_range) {
continue;
}
request.path = std::move(*path);
request.frame_start = frame_range->first();
request.frame_end = frame_range->last();
@ -620,6 +710,44 @@ static bool bake_directory_has_data(const StringRefNull absolute_bake_dir)
return true;
}
static bool may_have_disk_bake(const NodesModifierData &nmd)
{
if (nmd.bake_target == NODES_MODIFIER_BAKE_TARGET_DISK) {
return true;
}
for (const NodesModifierBake &bake : Span{nmd.bakes, nmd.bakes_num}) {
if (bake.bake_target == NODES_MODIFIER_BAKE_TARGET_DISK) {
return true;
}
}
return false;
}
static void initialize_modifier_bake_directory_if_necessary(bContext *C,
Object &object,
NodesModifierData &nmd,
wmOperator *op)
{
const bool bake_directory_set = !StringRef(nmd.bake_directory).is_empty();
if (bake_directory_set) {
return;
}
if (!may_have_disk_bake(nmd)) {
return;
}
Main *bmain = CTX_data_main(C);
BKE_reportf(op->reports,
RPT_INFO,
"Bake directory of object %s, modifier %s is empty, setting default path",
object.id.name + 2,
nmd.modifier.name);
nmd.bake_directory = BLI_strdup(
bake::get_default_modifier_bake_directory(*bmain, object, nmd).c_str());
}
static void bake_simulation_validate_paths(bContext *C,
wmOperator *op,
const Span<Object *> objects)
@ -635,18 +763,8 @@ static void bake_simulation_validate_paths(bContext *C,
if (md->type != eModifierType_Nodes) {
continue;
}
NodesModifierData *nmd = reinterpret_cast<NodesModifierData *>(md);
if (StringRef(nmd->bake_directory).is_empty()) {
BKE_reportf(op->reports,
RPT_INFO,
"Bake directory of object %s, modifier %s is empty, setting default path",
object->id.name + 2,
md->name);
nmd->bake_directory = BLI_strdup(
bake::get_default_modifier_bake_directory(*bmain, *object, *nmd).c_str());
}
initialize_modifier_bake_directory_if_necessary(C, *object, *nmd, op);
}
}
}
@ -702,7 +820,7 @@ static int bake_simulation_invoke(bContext *C, wmOperator *op, const wmEvent * /
}
}
/* Set empty paths to default. */
/* Set empty paths to default if necessary. */
bake_simulation_validate_paths(C, op, objects);
PathUsersMap path_users = bake_simulation_get_path_users(C, objects);
@ -815,10 +933,7 @@ static Vector<NodeBakeRequest> bake_single_node_gather_bake_request(bContext *C,
return {};
}
if (StringRef(nmd.bake_directory).is_empty()) {
const std::string directory = bake::get_default_modifier_bake_directory(*bmain, *object, nmd);
nmd.bake_directory = BLI_strdup(directory.c_str());
}
initialize_modifier_bake_directory_if_necessary(C, *object, nmd, op);
const int bake_id = RNA_int_get(op->ptr, "bake_id");
const bNode *node = nmd.node_group->find_nested_node(bake_id);
@ -840,13 +955,9 @@ static Vector<NodeBakeRequest> bake_single_node_gather_bake_request(bContext *C,
if (!bake) {
return {};
}
const std::optional<bake::BakePath> bake_path = bake::get_node_bake_path(
*bmain, *object, nmd, bake_id);
if (!bake_path.has_value()) {
BKE_report(op->reports, RPT_ERROR, "Cannot determine bake location on disk");
return {};
if (bake::get_node_bake_target(*object, nmd, bake_id) == NODES_MODIFIER_BAKE_TARGET_DISK) {
request.path = bake::get_node_bake_path(*bmain, *object, nmd, bake_id);