use multi-function

release/scripts/startup/nodeitems_builtins.py

@@ -754,6 +754,7 @@ geometry_node_categories = [
         NodeItem("GeometryNodeSwitch"),
         NodeItem("FunctionNodeRandomValue"),
         NodeItem("FunctionNodeAlignEulerToVector"),
+        NodeItem("GeometryNodeSampleSound"),
     ]),
     GeometryNodeCategory("GEO_VECTOR", "Vector", items=[
         NodeItem("ShaderNodeVectorCurve"),

source/blender/blenkernel/BKE_node.h

@@ -1553,6 +1553,7 @@ int ntreeTexExecTree(struct bNodeTree *ntree,
 #define GEO_NODE_VOLUME_TO_MESH 1133
 #define GEO_NODE_INPUT_ID 1134
 #define GEO_NODE_SET_ID 1135
+#define GEO_NODE_SAMPLE_SOUND 1136
 
 /** \} */
 

source/blender/blenkernel/intern/node.cc

@@ -5929,6 +5929,7 @@ static void registerGeometryNodes()
   register_node_type_geo_raycast();
   register_node_type_geo_realize_instances();
   register_node_type_geo_rotate_instances();
+  register_node_type_geo_sample_sound();
   register_node_type_geo_sample_texture();
   register_node_type_geo_scale_instances();
   register_node_type_geo_separate_components();

source/blender/blenkernel/intern/sound.c

@@ -86,6 +86,8 @@ static void sound_copy_data(Main *UNUSED(bmain),
   sound_dst->playback_handle = NULL;
   sound_dst->spinlock = MEM_mallocN(sizeof(SpinLock), "sound_spinlock");
   BLI_spin_init(sound_dst->spinlock);
+  sound_dst->tot_samples = 0;
+  sound_dst->samples = NULL;
 
   /* Just to be sure, should not have any value actually after reading time. */
   sound_dst->ipo = NULL;
@@ -117,6 +119,9 @@ static void sound_free_data(ID *id)
     MEM_freeN(sound->spinlock);
     sound->spinlock = NULL;
   }
+  if (sound->samples) {
+    AUD_Sound_freeData(sound->samples);
+  }
 }
 
 static void sound_foreach_cache(ID *id,
@@ -162,6 +167,8 @@ static void sound_blend_read_data(BlendDataReader *reader, ID *id)
   sound->tags = 0;
   sound->handle = NULL;
   sound->playback_handle = NULL;
+  sound->samples = NULL;
+  sound->tot_samples = 0;
 
   /* versioning stuff, if there was a cache, then we enable caching: */
   if (sound->cache) {

source/blender/makesdna/DNA_sound_types.h

@@ -93,6 +93,11 @@ typedef struct bSound {
   void *spinlock;
   /* XXX unused currently (SOUND_TYPE_LIMITER) */
   /* float start, end; */
+
+  float *samples;
+  int tot_samples;
+  char _pad2[4];
+  void *_pad3;
 } bSound;
 
 /* XXX unused currently */

source/blender/makesrna/intern/rna_nodetree.c

@@ -11091,6 +11091,18 @@ static void def_geo_viewer(StructRNA *srna)
   RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_GeometryNode_socket_update");
 }
 
+static void def_geo_sample_sound(StructRNA *srna)
+{
+  PropertyRNA *prop;
+
+  prop = RNA_def_property(srna, "sound", PROP_POINTER, PROP_NONE);
+  RNA_def_property_pointer_sdna(prop, NULL, "id");
+  RNA_def_property_struct_type(prop, "Sound");
+  RNA_def_property_flag(prop, PROP_EDITABLE);
+  RNA_def_property_ui_text(prop, "Sound", "");
+  RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_Node_update");
+}
+
 /* -------------------------------------------------------------------------- */
 
 static void rna_def_shader_node(BlenderRNA *brna)

source/blender/nodes/CMakeLists.txt

@@ -268,6 +268,7 @@ set(SRC
   geometry/nodes/node_geo_raycast.cc
   geometry/nodes/node_geo_realize_instances.cc
   geometry/nodes/node_geo_rotate_instances.cc
+  geometry/nodes/node_geo_sample_sound.cc
   geometry/nodes/node_geo_scale_instances.cc
   geometry/nodes/node_geo_separate_components.cc
   geometry/nodes/node_geo_separate_geometry.cc
@@ -552,4 +553,19 @@ if(WITH_OPENVDB)
   add_definitions(-DWITH_OPENVDB ${OPENVDB_DEFINITIONS})
 endif()
 
+if(WITH_AUDASPACE)
+  add_definitions(-DWITH_AUDASPACE)
+
+  list(APPEND INC_SYS
+    ${AUDASPACE_C_INCLUDE_DIRS}
+  )
+endif()
+
+if(WITH_FFTW3)
+list(APPEND INC_SYS
+    ${FFTW3_INCLUDE_DIRS}
+  )
+  add_definitions(-DWITH_FFTW3)
+endif()
+
 blender_add_lib(bf_nodes "${SRC}" "${INC}" "${INC_SYS}" "${LIB}")

source/blender/nodes/NOD_geometry.h

@@ -140,6 +140,7 @@ void register_node_type_geo_proximity(void);
 void register_node_type_geo_raycast(void);
 void register_node_type_geo_realize_instances(void);
 void register_node_type_geo_rotate_instances(void);
+void register_node_type_geo_sample_sound(void);
 void register_node_type_geo_sample_texture(void);
 void register_node_type_geo_scale_instances(void);
 void register_node_type_geo_select_by_handle_type(void);

source/blender/nodes/NOD_static_types.h

@@ -418,6 +418,7 @@ DefNode(GeometryNode, GEO_NODE_TRIANGULATE, def_geo_triangulate, "TRIANGULATE",
 DefNode(GeometryNode, GEO_NODE_TRIM_CURVE, def_geo_curve_trim, "TRIM_CURVE", TrimCurve, "Trim Curve", "")
 DefNode(GeometryNode, GEO_NODE_VIEWER, def_geo_viewer, "VIEWER", Viewer, "Viewer", "")
 DefNode(GeometryNode, GEO_NODE_VOLUME_TO_MESH, def_geo_volume_to_mesh, "VOLUME_TO_MESH", VolumeToMesh, "Volume to Mesh", "")
+DefNode(GeometryNode, GEO_NODE_SAMPLE_SOUND, def_geo_sample_sound, "SAMPLE_SOUND", SampleSound, "Sample Sound", "")
 
 /* undefine macros */
 #undef DefNode

source/blender/nodes/geometry/nodes/node_geo_sample_sound.cc

@@ -0,0 +1,189 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <fftw3.h>
+
+#include "UI_interface.h"
+#include "UI_resources.h"
+
+#include "BLI_double2.hh"
+
+#include "DNA_sound_types.h"
+
+#include "AUD_Sound.h"
+
+#include "BKE_sound.h"
+
+#include "DEG_depsgraph_query.h"
+
+#include "node_geometry_util.hh"
+
+namespace blender::nodes {
+
+static void geo_node_sample_sound_declare(NodeDeclarationBuilder &b)
+{
+  b.add_input<decl::Float>(N_("Time")).supports_field();
+  b.add_input<decl::Float>(N_("Min Frequency")).supports_field().default_value(0.0f).min(0.0f);
+  b.add_input<decl::Float>(N_("Max Frequency")).supports_field().default_value(20000.0f).min(0.0f);
+  b.add_input<decl::Int>(N_("Channel")).min(0);
+  b.add_output<decl::Float>(N_("Volume")).dependent_field();
+}
+
+static void geo_node_sample_sound_layout(uiLayout *layout, bContext *UNUSED(C), PointerRNA *ptr)
+{
+  uiItemR(layout, ptr, "sound", 0, "", ICON_NONE);
+}
+
+struct SoundData {
+  Span<float> samples;
+  float sample_rate;
+  int channels;
+};
+
+class SampleSoundFunction : public fn::MultiFunction {
+ private:
+  SoundData sound_data_;
+
+ public:
+  SampleSoundFunction(SoundData sound_data) : sound_data_(sound_data)
+  {
+    static fn::MFSignature signature = create_signature();
+    this->set_signature(&signature);
+  }
+
+  static fn::MFSignature create_signature()
+  {
+    fn::MFSignatureBuilder signature{"Sample Sound"};
+    signature.single_input<float>("Time");
+    signature.single_input<float>("Min Frequency");
+    signature.single_input<float>("Max Frequency");
+    signature.single_input<int>("Channel");
+    signature.single_output<float>("Volume");
+    return signature.build();
+  }
+
+  void call(IndexMask mask, fn::MFParams params, fn::MFContext UNUSED(context)) const override
+  {
+    const VArray<float> &times = params.readonly_single_input<float>(0, "Time");
+    const VArray<float> &min_frequencies = params.readonly_single_input<float>(1,
+                                                                               "Min Frequencies");
+    const VArray<float> &max_frequencies = params.readonly_single_input<float>(2,
+                                                                               "Max Frequencies");
+    const VArray<int> &channels = params.readonly_single_input<int>(3, "Channel");
+    MutableSpan<float> r_volumes = params.uninitialized_single_output<float>(4, "Volume");
+
+    /* TODO: Remove this limitation. */
+    BLI_assert(times.is_single());
+    BLI_assert(channels.is_single());
+
+    if (mask.is_empty()) {
+      return;
+    }
+
+    const float time = times.get_internal_single();
+    const int channel = channels.get_internal_single();
+
+    const int desired_slice_size = 4000;
+    int end_sample = time * sound_data_.sample_rate;
+    int start_sample = end_sample - desired_slice_size;
+    CLAMP(start_sample, 0, sound_data_.samples.size());
+    CLAMP(end_sample, 0, sound_data_.samples.size());
+    const int slice_size = end_sample - start_sample;
+
+    if (slice_size == 0) {
+      r_volumes.fill_indices(mask, 0.0f);
+      return;
+    }
+
+    Array<double> raw_samples(slice_size);
+    Array<double2> frequencies(slice_size / 2 + 1);
+
+    fftw_plan plan = fftw_plan_dft_r2c_1d(
+        slice_size, raw_samples.data(), (fftw_complex *)frequencies.data(), 0);
+
+    for (const int i : IndexRange(slice_size)) {
+      raw_samples[i] = sound_data_.samples[(start_sample + i) * sound_data_.channels + channel];
+    }
+
+    fftw_execute(plan);
+    fftw_destroy_plan(plan);
+
+    const float band_per_index = sound_data_.sample_rate / slice_size;
+    auto frequency_to_index = [&](const float frequency) -> int {
+      return frequency / band_per_index;
+    };
+
+    for (const int i : mask) {
+      const float min_frequency = min_frequencies[i];
+      const float max_frequency = max_frequencies[i];
+      int min_index = frequency_to_index(min_frequency);
+      int max_index = frequency_to_index(max_frequency);
+      min_index = std::clamp<int>(min_index, 0, frequencies.size());
+      max_index = std::clamp<int>(max_index, min_index, frequencies.size());
+
+      float sum = 0.0f;
+      for (int frequency_index = min_index; frequency_index < max_index; frequency_index++) {
+        sum += fabsf(frequencies[frequency_index].x);
+      }
+      const float average = (max_index > min_index) ? sum / (max_index - min_index) : 0.0f;
+      r_volumes[i] = average;
+    }
+  }
+};
+
+static void geo_node_sample_sound_exec(GeoNodeExecParams params)
+{
+  bSound *sound_id = (bSound *)params.node().id;
+  if (sound_id != nullptr) {
+    AUD_Sound *sound = (AUD_Sound *)sound_id->handle;
+    if (sound != NULL) {
+      if (sound_id->samples == NULL) {
+        AUD_Specs specs;
+        sound_id->samples = AUD_Sound_data(sound, &sound_id->tot_samples, &specs);
+      }
+      AUD_Specs specs = AUD_Sound_getSpecs(sound);
+      SoundData sound_data;
+      sound_data.channels = specs.channels;
+      sound_data.sample_rate = specs.rate;
+      sound_data.samples = Span<float>(sound_id->samples, sound_id->tot_samples);
+      auto sample_fn = std::make_shared<SampleSoundFunction>(sound_data);
+      auto sample_node = std::make_shared<FieldOperation>(
+          sample_fn,
+          Vector<GField>{params.extract_input<Field<float>>("Time"),
+                         params.extract_input<Field<float>>("Min Frequency"),
+                         params.extract_input<Field<float>>("Max Frequency"),
+                         params.extract_input<Field<int>>("Channel")});
+      Field<float> output_field{std::move(sample_node), 0};
+      params.set_output("Volume", std::move(output_field));
+      return;
+    }
+  }
+  params.set_output("Volume", 0.0f);
+}
+
+}  // namespace blender::nodes
+
+void register_node_type_geo_sample_sound()
+{
+  static bNodeType ntype;
+
+  geo_node_type_base(&ntype, GEO_NODE_SAMPLE_SOUND, "Sample Sound", NODE_CLASS_TEXTURE, 0);
+  node_type_size(&ntype, 200, 40, 1000);
+  ntype.declare = blender::nodes::geo_node_sample_sound_declare;
+  ntype.geometry_node_execute = blender::nodes::geo_node_sample_sound_exec;
+  ntype.draw_buttons = blender::nodes::geo_node_sample_sound_layout;
+  nodeRegisterType(&ntype);
+}