Fix T77685: object transforms from rigid body simulation are ignored by modifiers

This does not fix all the cases in the bug report, because there are multiple
different issues. Only the first two are fixed. The third is probably a known
issue for now.

Before this patch, the rigid body simulation was always done after modifiers
are evaluated, because to perform the simulation, the final geometry of the
object was required. However, the geometry is not required in all cases,
depending on the selected collisions shape.

This patch changes it so that when the simulation does not need the
evaluated geometry, the simulation will be done before the modifiers
are evaluated. This gives the modifiers access to the simulated positions.
When the rigid body simulation does depend on the evaluated geometry,
it will still be performed after modifiers are evaluated.

The simulation will be performed after modifiers are evaluated, iff
the collision shape is "Convex Hull" or "Mesh" and the source is set
to "Deform" or "Final".

Reviewers: sergey

Differential Revision: https://developer.blender.org/D8487

source/blender/depsgraph/intern/builder/deg_builder_relations.cc

@@ -1013,6 +1013,9 @@ void DepsgraphRelationBuilder::build_object_pointcache(Object *object)
     /* Check which components needs the point cache. */
     int flag = -1;
     if (ptcache_id->type == PTCACHE_TYPE_RIGIDBODY) {
+      if (object->rigidbody_object->type == RBO_TYPE_PASSIVE) {
+        continue;
+      }
       flag = FLAG_TRANSFORM;
       OperationKey transform_key(
           &object->id, NodeType::TRANSFORM, OperationCode::TRANSFORM_SIMULATION_INIT);
@@ -1713,8 +1716,7 @@ void DepsgraphRelationBuilder::build_rigidbody(Scene *scene)
       /* Geometry must be known to create the rigid body. RBO_MESH_BASE
        * uses the non-evaluated mesh, so then the evaluation is
        * unnecessary. */
-      if (object->rigidbody_object != nullptr &&
-          object->rigidbody_object->mesh_source != RBO_MESH_BASE) {
+      if (rigidbody_object_depends_on_evaluated_geometry(object->rigidbody_object)) {
         /* NOTE: We prefer this relation to be never killed, to avoid
          * access partially evaluated mesh from solver. */
         ComponentKey object_geometry_key(&object->id, NodeType::GEOMETRY);

source/blender/depsgraph/intern/builder/deg_builder_relations_impl.h

@@ -27,6 +27,7 @@
 
 #include "DNA_ID.h"
 #include "DNA_object_types.h"
+#include "DNA_rigidbody_types.h"
 
 namespace blender {
 namespace deg {
@@ -126,6 +127,19 @@ Relation *DepsgraphRelationBuilder::add_node_handle_relation(const KeyType &key_
   return nullptr;
 }
 
+static bool rigidbody_object_depends_on_evaluated_geometry(const RigidBodyOb *rbo)
+{
+  if (rbo == nullptr) {
+    return false;
+  }
+  if (ELEM(rbo->shape, RB_SHAPE_CONVEXH, RB_SHAPE_TRIMESH)) {
+    if (rbo->mesh_source != RBO_MESH_BASE) {
+      return true;
+    }
+  }
+  return false;
+}
+
 template<typename KeyTo>
 Relation *DepsgraphRelationBuilder::add_depends_on_transform_relation(ID *id,
                                                                       const KeyTo &key_to,
@@ -134,7 +148,7 @@ Relation *DepsgraphRelationBuilder::add_depends_on_transform_relation(ID *id,
 {
   if (GS(id->name) == ID_OB) {
     Object *object = reinterpret_cast<Object *>(id);
-    if (object->rigidbody_object != nullptr) {
+    if (rigidbody_object_depends_on_evaluated_geometry(object->rigidbody_object)) {
       OperationKey transform_key(&object->id, NodeType::TRANSFORM, OperationCode::TRANSFORM_EVAL);
       return add_relation(transform_key, key_to, description, flags);
     }

source/blender/makesrna/intern/rna_rigidbody.c

@@ -35,6 +35,8 @@
 #include "BLI_math.h"
 #include "BLI_utildefines.h"
 
+#include "DEG_depsgraph_build.h"
+
 #include "WM_types.h"
 
 /* roles of objects in RigidBody Sims */
@@ -218,6 +220,7 @@ static void rna_RigidBodyOb_shape_update(Main *bmain, Scene *scene, PointerRNA *
   Object *ob = (Object *)ptr->owner_id;
 
   rna_RigidBodyOb_reset(bmain, scene, ptr);
+  DEG_relations_tag_update(bmain);
 
   WM_main_add_notifier(NC_OBJECT | ND_DRAW, ob);
 }
@@ -233,6 +236,16 @@ static void rna_RigidBodyOb_shape_reset(Main *UNUSED(bmain), Scene *scene, Point
   }
 }
 
+static void rna_RigidBodyOb_mesh_source_update(Main *bmain, Scene *scene, PointerRNA *ptr)
+{
+  Object *ob = (Object *)ptr->owner_id;
+
+  rna_RigidBodyOb_reset(bmain, scene, ptr);
+  DEG_relations_tag_update(bmain);
+
+  WM_main_add_notifier(NC_OBJECT | ND_DRAW, ob);
+}
+
 static char *rna_RigidBodyOb_path(PointerRNA *UNUSED(ptr))
 {
   /* NOTE: this hardcoded path should work as long as only Objects have this */
@@ -1026,7 +1039,7 @@ static void rna_def_rigidbody_object(BlenderRNA *brna)
   RNA_def_property_ui_text(
       prop, "Mesh Source", "Source of the mesh used to create collision shape");
   RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
-  RNA_def_property_update(prop, NC_OBJECT | ND_POINTCACHE, "rna_RigidBodyOb_reset");
+  RNA_def_property_update(prop, NC_OBJECT | ND_POINTCACHE, "rna_RigidBodyOb_mesh_source_update");
 
   /* booleans */
   prop = RNA_def_property(srna, "enabled", PROP_BOOLEAN, PROP_NONE);