Bugfix #25652

Report was that move-to-layer menu failed. The real cause was
more complex; had to dive deep in the dungeons of the interface
code that handled undos and operators. Found several issues:

- popup menus (like redo operator, color picker) executed again
  on a mouse-exit
- far too many buttons were sending undo pushes; even worse, in
  the operator redo-panel each button action was pushed twice
- in case operator redo-buttons have own callbacks (like layer
  buttons) the redo wasn't working
- layerbutton menu was called without creating a proper undo/redo
  case

Things should all work smoother now!

On todo:
- better definition and handling of all versions for operator menus
  (four types now, not fun)
  also: make operator "do" menu, which on first action does operator
  and then switches to redo-ing
- bring back Undo menu, to list the undo stack and jump in it.

source/blender/editors/include/UI_interface.h

@@ -211,6 +211,7 @@ typedef struct uiLayout uiLayout;
 #define TOGBUT		(37<<9)
 #define OPTION		(38<<9)
 #define OPTIONN		(39<<9)
+		/* buttons with value >= SEARCH_MENU don't get undo pushes */
 #define SEARCH_MENU	(40<<9)
 #define BUT_EXTRA	(41<<9)
 #define HSVCIRCLE	(42<<9)

source/blender/editors/interface/interface.c

@@ -2505,8 +2505,10 @@ static uiBut *ui_def_but(uiBlock *block, int type, int retval, const char *str,
 		}
 	}
 
-	if(ELEM8(but->type, BLOCK, BUT, LABEL, PULLDOWN, ROUNDBOX, LISTBOX, SEARCH_MENU, BUTM));
+	/* keep track of UI_interface.h */
+	if(ELEM7(but->type, BLOCK, BUT, LABEL, PULLDOWN, ROUNDBOX, LISTBOX, BUTM));
 	else if(ELEM5(but->type, SCROLL, SEPR, LINK, INLINK, FTPREVIEW));
+	else if(but->type >= SEARCH_MENU);
 	else but->flag |= UI_BUT_UNDO;
 
 	BLI_addtail(&block->buttons, but);

source/blender/editors/interface/interface_handlers.c

@@ -5722,7 +5722,7 @@ int ui_handle_menu_event(bContext *C, wmEvent *event, uiPopupBlockHandle *menu,
 
 				if(ELEM3(event->type, LEFTMOUSE, MIDDLEMOUSE, RIGHTMOUSE) && event->val==KM_PRESS) {
 					if(saferct && !BLI_in_rctf(&saferct->parent, event->x, event->y)) {
-						if(block->flag & (UI_BLOCK_OUT_1|UI_BLOCK_KEEP_OPEN))
+						if(block->flag & (UI_BLOCK_OUT_1))
 							menu->menuretval= UI_RETURN_OK;
 						else
 							menu->menuretval= UI_RETURN_OUT;
@@ -5763,7 +5763,7 @@ int ui_handle_menu_event(bContext *C, wmEvent *event, uiPopupBlockHandle *menu,
 
 					/* strict check, and include the parent rect */
 					if(!menu->dotowards && !saferct) {
-						if(block->flag & (UI_BLOCK_OUT_1|UI_BLOCK_KEEP_OPEN))
+						if(block->flag & (UI_BLOCK_OUT_1))
 							menu->menuretval= UI_RETURN_OK;
 						else
 							menu->menuretval= UI_RETURN_OUT;

source/blender/editors/interface/interface_layout.c

@@ -1234,7 +1234,7 @@ void ui_but_add_search(uiBut *but, PointerRNA *ptr, PropertyRNA *prop, PointerRN
 		but->hardmax= MAX2(but->hardmax, 256);
 		but->rnasearchpoin= *searchptr;
 		but->rnasearchprop= searchprop;
-		but->flag |= UI_ICON_LEFT|UI_TEXT_LEFT|UI_BUT_UNDO;
+		but->flag |= UI_ICON_LEFT|UI_TEXT_LEFT;
 
 		uiButSetSearchFunc(but, rna_search_cb, but, NULL, NULL);
 	}
@@ -2730,4 +2730,12 @@ void uiLayoutOperatorButs(const bContext *C, uiLayout *layout, wmOperator *op,in
 			uiItemL(layout, "No Properties.", ICON_NULL);
 		}
 	}
+	
+	/* no undo for buttons for operator redo panels */
+	{
+		uiBut *but;
+		
+		for(but= uiLayoutGetBlock(layout)->buttons.first; but; but= but->next)
+			uiButClearFlag(but, UI_BUT_UNDO);
+	}
 }

source/blender/editors/interface/interface_regions.c

@@ -1984,7 +1984,7 @@ uiBlock *ui_block_func_COL(bContext *C, uiPopupBlockHandle *handle, void *arg_bu
 	
 	uiBlockPicker(block, handle->retvec, &but->rnapoin, but->rnaprop);
 	
-	block->flag= UI_BLOCK_LOOP|UI_BLOCK_REDRAW|UI_BLOCK_KEEP_OPEN;
+	block->flag= UI_BLOCK_LOOP|UI_BLOCK_REDRAW|UI_BLOCK_KEEP_OPEN|UI_BLOCK_OUT_1;
 	uiBoundsBlock(block, 10);
 	
 	block->block_event_func= ui_picker_small_wheel_cb;

source/blender/editors/space_view3d/view3d_toolbar.c

@@ -118,7 +118,8 @@ static void view3d_panel_operator_redo(const bContext *C, Panel *pa)
 	if(ED_undo_valid(C, op->type->name)==0)
 		uiLayoutSetEnabled(pa->layout, 0);
 
-	uiBlockSetFunc(block, ED_undo_operator_repeat_cb, op, NULL);
+	/* note, blockfunc is a default but->func, use Handle func to allow button callbacks too */
+	uiBlockSetHandleFunc(block, ED_undo_operator_repeat_cb_evt, op);
 	
 	view3d_panel_operator_redo_operator(C, pa, op);
 }

source/blender/windowmanager/intern/wm_operators.c

@@ -981,25 +981,21 @@ static uiBlock *wm_operator_create_ui(bContext *C, ARegion *ar, void *userData)
 	return block;
 }
 
+/* operator menu needs undo, for redo callback */
 int WM_operator_props_popup(bContext *C, wmOperator *op, wmEvent *UNUSED(event))
 {
-	int retval= OPERATOR_CANCELLED;
 	
 	if((op->type->flag & OPTYPE_REGISTER)==0) {
 		BKE_reportf(op->reports, RPT_ERROR, "Operator '%s' does not have register enabled, incorrect invoke function.", op->type->idname);
 		return OPERATOR_CANCELLED;
 	}
 	
-	if(op->type->exec) {
+	ED_undo_push_op(C, op);
-		retval= op->type->exec(C, op);
+	wm_operator_register(C, op);
 
-		/* ED_undo_push_op(C, op), called by wm_operator_finished now. */
-	}
-
-	if(retval != OPERATOR_CANCELLED)
 	uiPupBlock(C, wm_block_create_redo, op);
 
-	return retval;
+	return OPERATOR_RUNNING_MODAL;
 }
 
 int WM_operator_props_dialog_popup(bContext *C, wmOperator *op, int width, int height)