Tool System: make smooth & randomize modal operators

Previously these used a gizmo to redo the operator however this
complicated having on-screen gizmos to access tools (see T66304).

Replace this with a generic way to make an operator that only has an
execute function into a modal operator.

This is used for smooth and randomize tools.

Unlike operator gestures, this handles storing and resetting the data.

Currently this only handles edit-mode data, however it's can be
extended to other kinds of data.

release/scripts/presets/keyconfig/keymap_data/blender_default.py

@@ -5527,7 +5527,7 @@ def km_3d_view_tool_edit_mesh_smooth(params):
         {"space_type": 'VIEW_3D', "region_type": 'WINDOW'},
         {"items": [
             ("mesh.vertices_smooth", {"type": params.tool_tweak, "value": 'ANY'},
-             {"properties": [("factor", 0.0)]}),
+             {"properties": [("factor", 0.0), ("wait_for_input", False)]}),
         ]},
     )
 
@@ -5538,7 +5538,7 @@ def km_3d_view_tool_edit_mesh_randomize(params):
         {"space_type": 'VIEW_3D', "region_type": 'WINDOW'},
         {"items": [
             ("transform.vertex_random", {"type": params.tool_tweak, "value": 'ANY'},
-             {"properties": [("offset", 0.0)]}),
+             {"properties": [("offset", 0.0), ("wait_for_input", False)]}),
         ]},
     )
 

release/scripts/startup/bl_ui/space_toolsystem_toolbar.py

@@ -698,7 +698,7 @@ class _defs_edit_mesh:
             idname="builtin.smooth",
             label="Smooth",
             icon="ops.mesh.vertices_smooth",
-            widget="WM_GGT_value_operator_redo",
+            widget=None,
             keymap=(),
             draw_settings=draw_settings,
         )
@@ -714,7 +714,7 @@ class _defs_edit_mesh:
             idname="builtin.randomize",
             label="Randomize",
             icon="ops.transform.vertex_random",
-            widget="WM_GGT_value_operator_redo",
+            widget=None,
             keymap=(),
             draw_settings=draw_settings,
         )

release/scripts/startup/bl_ui/space_view3d.py

@@ -830,7 +830,9 @@ class VIEW3D_MT_transform_base(Menu):
 
         if context.mode != 'OBJECT':
             layout.operator("transform.vertex_warp", text="Warp")
+            layout.operator_context = 'EXEC_DEFAULT'
             layout.operator("transform.vertex_random", text="Randomize")
+            layout.operator_context = 'INVOKE_REGION_WIN'
 
 
 # Generic transform menu - geometry types
@@ -3425,8 +3427,10 @@ class VIEW3D_MT_edit_mesh_context_menu(Menu):
             col.operator("transform.shrink_fatten", text="Shrink/Fatten")
             col.operator("transform.shear", text="Shear")
             col.operator("transform.vert_slide", text="Slide Vertices")
+            col.operator_context = 'EXEC_DEFAULT'
             col.operator("transform.vertex_random", text="Randomize Vertices")
             col.operator("mesh.vertices_smooth", text="Smooth Vertices")
+            col.operator_context = 'INVOKE_REGION_WIN'
             col.operator("mesh.vertices_smooth_laplacian", text="Smooth Laplacian")
 
             col.separator()
@@ -3640,7 +3644,9 @@ class VIEW3D_MT_edit_mesh_vertices(Menu):
         layout.separator()
 
         layout.operator("transform.vert_slide", text="Slide Vertices")
+        layout.operator_context = 'EXEC_DEFAULT'
         layout.operator("mesh.vertices_smooth", text="Smooth Vertices")
+        layout.operator_context = 'INVOKE_REGION_WIN'
 
         layout.separator()
 

source/blender/editors/mesh/editmesh_tools.c

@@ -2366,6 +2366,9 @@ void MESH_OT_vertices_smooth(wmOperatorType *ot)
   RNA_def_boolean(ot->srna, "xaxis", true, "X-Axis", "Smooth along the X axis");
   RNA_def_boolean(ot->srna, "yaxis", true, "Y-Axis", "Smooth along the Y axis");
   RNA_def_boolean(ot->srna, "zaxis", true, "Z-Axis", "Smooth along the Z axis");
+
+  /* Set generic modal callbacks. */
+  WM_operator_type_modal_from_exec_for_object_edit_coords(ot);
 }
 
 /** \} */

source/blender/editors/object/object_random.c

@@ -180,4 +180,7 @@ void TRANSFORM_OT_vertex_random(struct wmOperatorType *ot)
                        1.0f);
   RNA_def_int(
       ot->srna, "seed", 0, 0, 10000, "Random Seed", "Seed for the random number generator", 0, 50);
+
+  /* Set generic modal callbacks. */
+  WM_operator_type_modal_from_exec_for_object_edit_coords(ot);
 }

source/blender/windowmanager/CMakeLists.txt

@@ -66,6 +66,7 @@ set(SRC
   intern/wm_menu_type.c
   intern/wm_operator_props.c
   intern/wm_operator_type.c
+  intern/wm_operator_utils.c
   intern/wm_operators.c
   intern/wm_panel_type.c
   intern/wm_platform_support.c

source/blender/windowmanager/WM_api.h

@@ -562,6 +562,9 @@ char *WM_operatortype_description(struct bContext *C,
                                   struct wmOperatorType *ot,
                                   struct PointerRNA *properties);
 
+/* wm_operator_utils.c */
+void WM_operator_type_modal_from_exec_for_object_edit_coords(struct wmOperatorType *ot);
+
 /* wm_uilist_type.c */
 void WM_uilisttype_init(void);
 struct uiListType *WM_uilisttype_find(const char *idname, bool quiet);

source/blender/windowmanager/intern/wm_operator_utils.c

@@ -0,0 +1,323 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup wm
+ *
+ * Utilities for Implementing Operators
+ */
+
+#include <math.h>
+
+#include "BLI_utildefines.h"
+#include "BLI_string.h"
+
+#include "BKE_context.h"
+#include "BKE_layer.h"
+
+#include "RNA_access.h"
+#include "RNA_define.h"
+
+#include "WM_api.h" /* Own include. */
+#include "WM_types.h"
+
+#include "MEM_guardedalloc.h"
+
+#include "ED_object.h"
+#include "ED_screen.h"
+
+/* -------------------------------------------------------------------- */
+/** \name Value Interaction Helper
+ *
+ * Possible additions (add as needed).
+ * - Int support.
+ * - Configurable motion (x/y).
+ *
+ * \{ */
+
+typedef struct ValueInteraction {
+  struct {
+    float mval[2];
+    float prop_value;
+  } init;
+  struct {
+    float prop_value;
+    bool is_snap;
+    bool is_precise;
+  } prev;
+  float range[2];
+
+  struct {
+    ScrArea *sa;
+    ARegion *ar;
+  } context_vars;
+} ValueInteraction;
+
+static void interactive_value_init(bContext *C,
+                                   ValueInteraction *inter,
+                                   const wmEvent *event,
+                                   const float value_final,
+                                   const float range[2])
+{
+
+  inter->context_vars.sa = CTX_wm_area(C);
+  inter->context_vars.ar = CTX_wm_region(C);
+
+  inter->init.mval[0] = event->mval[0];
+  inter->init.mval[1] = event->mval[1];
+  inter->init.prop_value = value_final;
+  inter->prev.prop_value = value_final;
+  inter->range[0] = range[0];
+  inter->range[1] = range[1];
+}
+
+static void interactive_value_init_from_property(
+    bContext *C, ValueInteraction *inter, const wmEvent *event, PointerRNA *ptr, PropertyRNA *prop)
+{
+  float range[2];
+  float step, precision;
+  RNA_property_float_ui_range(ptr, prop, &range[0], &range[1], &step, &precision);
+  const float value_final = RNA_property_float_get(ptr, prop);
+  interactive_value_init(C, inter, event, value_final, range);
+}
+
+static void interactive_value_exit(ValueInteraction *inter)
+{
+  ED_area_status_text(inter->context_vars.sa, NULL);
+}
+
+static bool interactive_value_update(ValueInteraction *inter,
+                                     const wmEvent *event,
+                                     float *r_value_final)
+{
+  const int mval_axis = 0;
+
+  const float value_scale = 4.0f; /* Could be option. */
+  const float value_range = inter->range[1] - inter->range[0];
+  const int mval_curr = event->mval[mval_axis];
+  const int mval_init = inter->init.mval[mval_axis];
+  float value_delta = (inter->init.prop_value +
+                       (((float)(mval_curr - mval_init) / inter->context_vars.ar->winx) *
+                        value_range)) *
+                      value_scale;
+  if (event->ctrl) {
+    const double snap = 0.1;
+    value_delta = (float)roundf((double)value_delta / snap) * snap;
+  }
+  if (event->shift) {
+    value_delta *= 0.1f;
+  }
+  const float value_final = inter->init.prop_value + value_delta;
+
+  const bool changed = value_final != inter->prev.prop_value;
+  if (changed) {
+    /* set the property for the operator and call its modal function */
+    char str[64];
+    SNPRINTF(str, "%.4f", value_final);
+    ED_area_status_text(inter->context_vars.sa, str);
+  }
+
+  inter->prev.prop_value = value_final;
+  inter->prev.is_snap = event->ctrl;
+  inter->prev.is_precise = event->shift;
+
+  *r_value_final = value_final;
+  return changed;
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Object Edit Mode Coords (Modal Callbacks)
+ *
+ * \note We could support object mode coords too, it's just not needed at the moment.
+ * \{ */
+
+struct ObCustomData_ForEditMode {
+  bool wait_for_input;
+  bool is_active;
+  bool is_first;
+
+  ValueInteraction inter;
+
+  /** This could be split into a sub-type if we support different kinds of data. */
+  Object **objects;
+  uint objects_len;
+  struct XFormObjectData **objects_xform;
+};
+
+/* Internal callback to free. */
+static void op_generic_value_exit(wmOperator *op)
+{
+  struct ObCustomData_ForEditMode *cd = op->customdata;
+  if (cd) {
+    interactive_value_exit(&cd->inter);
+
+    for (uint ob_index = 0; ob_index < cd->objects_len; ob_index++) {
+      struct XFormObjectData *xod = cd->objects_xform[ob_index];
+      if (xod != NULL) {
+        ED_object_data_xform_destroy(xod);
+      }
+    }
+    MEM_freeN(cd->objects);
+    MEM_freeN(cd->objects_xform);
+    MEM_freeN(cd);
+  }
+}
+
+static void op_generic_value_restore(wmOperator *op)
+{
+  struct ObCustomData_ForEditMode *cd = op->customdata;
+  for (uint ob_index = 0; ob_index < cd->objects_len; ob_index++) {
+    ED_object_data_xform_restore(cd->objects_xform[ob_index]);
+    ED_object_data_xform_tag_update(cd->objects_xform[ob_index]);
+  }
+}
+
+static void op_generic_value_cancel(bContext *UNUSED(C), wmOperator *op)
+{
+  op_generic_value_exit(op);
+}
+
+static int op_generic_value_invoke(bContext *C, wmOperator *op, const wmEvent *event)
+{
+  ViewLayer *view_layer = CTX_data_view_layer(C);
+  uint objects_len;
+  Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data(
+      view_layer, CTX_wm_view3d(C), &objects_len);
+  if (objects_len == 0) {
+    MEM_freeN(objects);
+    return OPERATOR_CANCELLED;
+  }
+
+  struct ObCustomData_ForEditMode *cd = MEM_callocN(sizeof(*cd), __func__);
+  cd->wait_for_input = RNA_boolean_get(op->ptr, "wait_for_input");
+  cd->is_active = !cd->wait_for_input;
+  cd->is_first = true;
+  cd->objects = objects;
+  cd->objects_len = objects_len;
+
+  if (cd->wait_for_input == false) {
+    interactive_value_init_from_property(C, &cd->inter, event, op->ptr, op->type->prop);
+  }
+
+  cd->objects_xform = MEM_callocN(sizeof(*cd->objects_xform) * objects_len, __func__);
+
+  for (uint ob_index = 0; ob_index < objects_len; ob_index++) {
+    Object *obedit = objects[ob_index];
+    cd->objects_xform[ob_index] = ED_object_data_xform_create_from_edit_mode(obedit->data);
+  }
+
+  op->customdata = cd;
+
+  WM_event_add_modal_handler(C, op);
+  return OPERATOR_RUNNING_MODAL;
+}
+
+static int op_generic_value_modal(bContext *C, wmOperator *op, const wmEvent *event)
+{
+  struct ObCustomData_ForEditMode *cd = op->customdata;
+  switch (event->type) {
+    case MOUSEMOVE:
+    case LEFTCTRLKEY:
+    case RIGHTCTRLKEY:
+    case LEFTSHIFTKEY:
+    case RIGHTSHIFTKEY: {
+      float value_final;
+      if (cd->is_active && interactive_value_update(&cd->inter, event, &value_final)) {
+        wmWindowManager *wm = CTX_wm_manager(C);
+
+        RNA_property_float_set(op->ptr, op->type->prop, value_final);
+        if (cd->is_first == false) {
+          op_generic_value_restore(op);
+        }
+
+        wm->op_undo_depth++;
+        int retval = op->type->exec(C, op);
+        OPERATOR_RETVAL_CHECK(retval);
+        wm->op_undo_depth--;
+
+        cd->is_first = false;
+
+        if ((retval & OPERATOR_FINISHED) == 0) {
+          op_generic_value_exit(op);
+          return OPERATOR_CANCELLED;
+        }
+      }
+      break;
+    }
+    case RETKEY:
+    case PADENTER:
+    case LEFTMOUSE: {
+      if (cd->wait_for_input) {
+        if (event->val == KM_PRESS) {
+          if (cd->is_active == false) {
+            cd->is_active = true;
+            interactive_value_init_from_property(C, &cd->inter, event, op->ptr, op->type->prop);
+          }
+        }
+        else if (event->val == KM_RELEASE) {
+          if (cd->is_active == true) {
+            op_generic_value_exit(op);
+            return OPERATOR_FINISHED;
+          }
+        }
+      }
+      else {
+        if (event->val == KM_RELEASE) {
+          op_generic_value_exit(op);
+          return OPERATOR_FINISHED;
+        }
+      }
+      break;
+    }
+    case ESCKEY:
+    case RIGHTMOUSE: {
+      if (event->val == KM_PRESS) {
+        if (cd->is_active == true) {
+          op_generic_value_restore(op);
+        }
+        op_generic_value_exit(op);
+        return OPERATOR_CANCELLED;
+      }
+      break;
+    }
+  }
+  return OPERATOR_RUNNING_MODAL;
+}
+
+/**
+ * Allow an operator with only and execute function to run modally,
+ * re-doing the action, using vertex coordinate store/restore instead of operator undo.
+ */
+void WM_operator_type_modal_from_exec_for_object_edit_coords(wmOperatorType *ot)
+{
+  PropertyRNA *prop;
+
+  BLI_assert(ot->modal == NULL);
+  BLI_assert(ot->invoke == NULL);
+  BLI_assert(ot->cancel == NULL);
+  BLI_assert(ot->prop != NULL);
+
+  ot->invoke = op_generic_value_invoke;
+  ot->modal = op_generic_value_modal;
+  ot->cancel = op_generic_value_cancel;
+
+  prop = RNA_def_boolean(ot->srna, "wait_for_input", true, "Wait for Input", "");
+  RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
+}
+
+/** \} */