Sculpt: Fix Dyntopo Warnings

Because of T95965, some attributes are stored as generic attributes
in Mesh but have special handling for the conversion to BMesh.

Expose a function to tell whether certain attribute names are handled
specially in the conversion, and refactor the error checking process
to use it. Also check for generic attributes on the face domain which
wasn't done before.

Author: Hans Goudey
Reviewed By: Joseph Eagar

Co-authored-by: Joseph Eagar <joeedh@gmail.com>
Pull Request #104567

source/blender/blenkernel/intern/customdata.cc

@@ -2289,26 +2289,12 @@ bool CustomData_merge(const CustomData *source,
   return changed;
 }
 
-static bool attribute_stored_in_bmesh_flag(const StringRef name)
-{
-  return ELEM(name,
-              "position",
-              ".hide_vert",
-              ".hide_edge",
-              ".hide_poly",
-              ".select_vert",
-              ".select_edge",
-              ".select_poly",
-              "material_index",
-              "sharp_edge");
-}
-
 CustomData CustomData_shallow_copy_remove_non_bmesh_attributes(const CustomData *src,
                                                                const eCustomDataMask mask)
 {
   Vector<CustomDataLayer> dst_layers;
   for (const CustomDataLayer &layer : Span<CustomDataLayer>{src->layers, src->totlayer}) {
-    if (attribute_stored_in_bmesh_flag(layer.name)) {
+    if (BM_attribute_stored_in_bmesh_builtin(layer.name)) {
       continue;
     }
     if (!(mask & CD_TYPE_AS_MASK(layer.type))) {

source/blender/bmesh/intern/bmesh_mesh_convert.cc

@@ -133,6 +133,20 @@ static char bm_face_flag_to_mflag(const BMFace *f)
   return ((hflag & BM_ELEM_SMOOTH) ? ME_SMOOTH : 0);
 }
 
+bool BM_attribute_stored_in_bmesh_builtin(const StringRef name)
+{
+  return ELEM(name,
+              "position",
+              ".hide_vert",
+              ".hide_edge",
+              ".hide_poly",
+              ".select_vert",
+              ".select_edge",
+              ".select_poly",
+              "material_index",
+              "sharp_edge");
+}
+
 /* Static function for alloc (duplicate in modifiers_bmesh.c) */
 static BMFace *bm_face_create_from_mpoly(BMesh &bm,
                                          Span<MLoop> loops,

source/blender/bmesh/intern/bmesh_mesh_convert.h

@@ -9,6 +9,16 @@
 
 #include "bmesh.h"
 
+#ifdef __cplusplus
+#  include "BLI_string_ref.hh"
+
+/**
+ * \return Whether attributes with the given name are stored in special flags or fields in BMesh
+ * rather than in the regular custom data blocks.
+ */
+bool BM_attribute_stored_in_bmesh_builtin(const blender::StringRef name);
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif

source/blender/editors/sculpt_paint/sculpt_dyntopo.cc

@@ -28,6 +28,8 @@
 #include "BKE_pointcache.h"
 #include "BKE_scene.h"
 
+#include "BLI_index_range.hh"
+
 #include "DEG_depsgraph.h"
 
 #include "WM_api.h"
@@ -42,6 +44,8 @@
 #include "bmesh.h"
 #include "bmesh_tools.h"
 
+using blender::IndexRange;
+
 void SCULPT_dynamic_topology_triangulate(BMesh *bm)
 {
   if (bm->totloop != bm->totface * 3) {
@@ -281,8 +285,8 @@ static int dyntopo_warning_popup(bContext *C, wmOperatorType *ot, enum eDynTopoW
   uiLayout *layout = UI_popup_menu_layout(pup);
 
   if (flag & (DYNTOPO_WARN_VDATA | DYNTOPO_WARN_EDATA | DYNTOPO_WARN_LDATA)) {
-    const char *msg_error = TIP_("Vertex Data Detected!");
-    const char *msg = TIP_("Dyntopo will not preserve vertex colors, UVs, or other customdata");
+    const char *msg_error = TIP_("Attribute Data Detected");
+    const char *msg = TIP_("Dyntopo will not preserve colors, UVs, or other attributes");
     uiItemL(layout, msg_error, ICON_INFO);
     uiItemL(layout, msg, ICON_NONE);
     uiItemS(layout);
@@ -305,6 +309,39 @@ static int dyntopo_warning_popup(bContext *C, wmOperatorType *ot, enum eDynTopoW
   return OPERATOR_INTERFACE;
 }
 
+static bool dyntopo_supports_customdata_layers(const blender::Span<CustomDataLayer> layers,
+                                               int totelem)
+{
+  for (const CustomDataLayer &layer : layers) {
+    if (CD_TYPE_AS_MASK(layer.type) & CD_MASK_PROP_ALL) {
+      if (layer.name[0] == '\0') {
+        return false;
+      }
+
+      if (STREQ(layer.name, ".sculpt_face_sets") && totelem > 0) {
+        int *fsets = static_cast<int *>(layer.data);
+        int fset = fsets[0];
+
+        /* Check if only one face set exists. */
+        for (int i : IndexRange(totelem)) {
+          if (fsets[i] != fset) {
+            return false;
+          }
+        }
+
+        return true;
+      }
+
+      /* Some data is stored as generic attributes on #Mesh but in flags or field on #BMesh. */
+      return BM_attribute_stored_in_bmesh_builtin(layer.name);
+    }
+    /* Some layers just encode #Mesh topology or are handled as special cases for dyntopo. */
+    return ELEM(layer.type, CD_MEDGE, CD_MFACE, CD_MLOOP, CD_MPOLY, CD_PAINT_MASK, CD_ORIGINDEX);
+  }
+
+  return true;
+}
+
 enum eDynTopoWarnFlag SCULPT_dynamic_topology_check(Scene *scene, Object *ob)
 {
   Mesh *me = static_cast<Mesh *>(ob->data);
@@ -315,18 +352,17 @@ enum eDynTopoWarnFlag SCULPT_dynamic_topology_check(Scene *scene, Object *ob)
   BLI_assert(ss->bm == nullptr);
   UNUSED_VARS_NDEBUG(ss);
 
-  for (int i = 0; i < CD_NUMTYPES; i++) {
-    if (!ELEM(i, CD_MEDGE, CD_MFACE, CD_MLOOP, CD_MPOLY, CD_PAINT_MASK, CD_ORIGINDEX)) {
-      if (CustomData_has_layer(&me->vdata, i)) {
-        flag |= DYNTOPO_WARN_VDATA;
-      }
-      if (CustomData_has_layer(&me->edata, i)) {
-        flag |= DYNTOPO_WARN_EDATA;
-      }
-      if (CustomData_has_layer(&me->ldata, i)) {
-        flag |= DYNTOPO_WARN_LDATA;
-      }
-    }
+  if (!dyntopo_supports_customdata_layers({me->vdata.layers, me->vdata.totlayer}, me->totvert)) {
+    flag |= DYNTOPO_WARN_VDATA;
+  }
+  if (!dyntopo_supports_customdata_layers({me->edata.layers, me->edata.totlayer}, me->totedge)) {
+    flag |= DYNTOPO_WARN_EDATA;
+  }
+  if (!dyntopo_supports_customdata_layers({me->pdata.layers, me->pdata.totlayer}, me->totpoly)) {
+    flag |= DYNTOPO_WARN_LDATA;
+  }
+  if (!dyntopo_supports_customdata_layers({me->ldata.layers, me->ldata.totlayer}, me->totloop)) {
+    flag |= DYNTOPO_WARN_LDATA;
   }
 
   {