Add partial visibility operator including keymaps and menu items.

Uses HKEY for border hide, CTRL+HKEY for border show, and ALT+HKEY for
show all.

Documentation:
http://wiki.blender.org/index.php/User:Nicholasbishop/PartialVisibility

Code review:
http://codereview.appspot.com/5695043

release/scripts/startup/bl_ui/space_view3d.py

@@ -54,6 +54,8 @@ class VIEW3D_HT_header(Header):
                     sub.menu("VIEW3D_MT_%s" % mode_string.lower())
                 if mode_string in {'SCULPT', 'PAINT_VERTEX', 'PAINT_WEIGHT', 'PAINT_TEXTURE'}:
                     sub.menu("VIEW3D_MT_brush")
+                if mode_string == 'SCULPT':
+                    sub.menu("VIEW3D_MT_hide")
             else:
                 sub.menu("VIEW3D_MT_object")
 
@@ -1195,6 +1197,25 @@ class VIEW3D_MT_sculpt(Menu):
         layout.prop(sculpt, "use_deform_only")
 
 
+class VIEW3D_MT_hide(Menu):
+    bl_label = "Hide"
+
+    def draw(self, context):
+        layout = self.layout
+
+        op = layout.operator("paint.hide_show", text="Show All")
+        op.action = 'SHOW'
+        op.area = 'ALL'
+
+        op = layout.operator("paint.hide_show", text="Hide Bounding Box")
+        op.action = 'HIDE'
+        op.area = 'INSIDE'
+
+        op = layout.operator("paint.hide_show", text="Show Bounding Box")
+        op.action = 'SHOW'
+        op.area = 'INSIDE'
+    
+
 # ********** Particle menu **********
 
 

source/blender/editors/sculpt_paint/CMakeLists.txt

@@ -40,6 +40,7 @@ set(INC_SYS
 
 set(SRC
 	paint_cursor.c
+	paint_hide.c
 	paint_image.c
 	paint_ops.c
 	paint_stroke.c

source/blender/editors/sculpt_paint/paint_hide.c

@@ -0,0 +1,392 @@
+/*
+ * $Id$
+ *
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software  Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2010 by Nicholas Bishop
+ * All rights reserved.
+ *
+ * The Original Code is: all of this file.
+ *
+ * Contributor(s):
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ *
+ * Implements the PBVH node hiding operator
+ *
+ */
+
+/** \file blender/editors/sculpt_paint/paint_hide.c
+ *  \ingroup edsculpt
+ */
+
+#include "MEM_guardedalloc.h"
+
+#include "BLI_bitmap.h"
+#include "BLI_listbase.h"
+#include "BLI_math_vector.h"
+#include "BLI_pbvh.h"
+#include "BLI_utildefines.h"
+
+#include "DNA_mesh_types.h"
+#include "DNA_meshdata_types.h"
+#include "DNA_object_types.h"
+#include "DNA_scene_types.h"
+
+#include "BKE_context.h"
+#include "BKE_DerivedMesh.h"
+#include "BKE_mesh.h"
+#include "BKE_multires.h"
+#include "BKE_paint.h"
+#include "BKE_subsurf.h"
+
+#include "BIF_glutil.h"
+
+#include "WM_api.h"
+#include "WM_types.h"
+
+#include "ED_screen.h"
+#include "ED_view3d.h"
+
+#include "RNA_access.h"
+#include "RNA_define.h"
+
+#include "paint_intern.h"
+#include "sculpt_intern.h" /* for undo push */
+
+#include <assert.h>
+
+static int planes_contain_v3(float (*planes)[4], int totplane, const float p[3])
+{
+	int i;
+
+	for(i = 0; i < totplane; i++) {
+		if(dot_v3v3(planes[i], p) + planes[i][3] > 0)
+			return 0;
+	}
+
+	return 1;
+}
+
+/* return true if the element should be hidden/shown */
+static int is_effected(PartialVisArea area,
+					   float planes[4][4],
+					   const float co[3])
+{
+	if(area == PARTIALVIS_ALL)
+		return 1;
+	else {
+		int inside = planes_contain_v3(planes, 4, co);
+		return ((inside && area == PARTIALVIS_INSIDE) ||
+				(!inside && area == PARTIALVIS_OUTSIDE));
+	}
+}
+
+static void partialvis_update_mesh(Object *ob,
+								   PBVH *pbvh,
+								   PBVHNode *node,
+								   PartialVisAction action,
+								   PartialVisArea area,
+								   float planes[4][4])
+{
+	MVert *mvert;
+	int *vert_indices;
+	int any_changed = 0, any_visible = 0, totvert, i;
+			
+	BLI_pbvh_node_num_verts(pbvh, node, NULL, &totvert);
+	BLI_pbvh_node_get_verts(pbvh, node, &vert_indices, &mvert);
+
+	sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
+
+	for(i = 0; i < totvert; i++) {
+		MVert *v = &mvert[vert_indices[i]];
+
+		/* hide vertex if in the hide volume */
+		if(is_effected(area, planes, v->co)) {
+			if(action == PARTIALVIS_HIDE)
+				v->flag |= ME_HIDE;
+			else
+				v->flag &= ~ME_HIDE;
+			any_changed = 1;
+		}
+
+		if(!(v->flag & ME_HIDE))
+			any_visible = 1;
+	}
+
+	if(any_changed) {
+		BLI_pbvh_node_mark_rebuild_draw(node);
+		BLI_pbvh_node_fully_hidden_set(node, !any_visible);
+	}
+}
+
+/* Hide or show elements in multires grids with a special GridFlags
+   customdata layer. */
+static void partialvis_update_grids(Object *ob,
+									PBVH *pbvh,
+									PBVHNode *node,
+									PartialVisAction action,
+									PartialVisArea area,
+									float planes[4][4])
+{
+	DMGridData **grids;
+	BLI_bitmap *grid_hidden;
+	int any_visible = 0;
+	int *grid_indices, gridsize, totgrid, any_changed, i;
+
+	/* get PBVH data */
+	BLI_pbvh_node_get_grids(pbvh, node,
+							&grid_indices, &totgrid, NULL, &gridsize,
+							&grids, NULL);
+	grid_hidden = BLI_pbvh_grid_hidden(pbvh);
+	
+	sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
+	
+	any_changed = 0;
+	for(i = 0; i < totgrid; i++) {
+		int any_hidden = 0;
+		int g = grid_indices[i], x, y;
+		BLI_bitmap gh = grid_hidden[g];
+
+		if(!gh) {
+			switch(action) {
+			case PARTIALVIS_HIDE:
+				/* create grid flags data */
+				gh = grid_hidden[g] = BLI_BITMAP_NEW(gridsize * gridsize,
+													 "partialvis_update_grids");
+				break;
+			case PARTIALVIS_SHOW:
+				/* entire grid is visible, nothing to show */
+				continue;
+			}
+		}
+		else if(action == PARTIALVIS_SHOW && area == PARTIALVIS_ALL) {
+			/* special case if we're showing all, just free the
+			   grid */
+			MEM_freeN(gh);
+			grid_hidden[g] = NULL;
+			any_changed = 1;
+			any_visible = 1;
+			continue;
+		}
+
+		for(y = 0; y < gridsize; y++) {
+			for(x = 0; x < gridsize; x++) {
+				const float *co = grids[g][y * gridsize + x].co;
+
+				/* skip grid element if not in the effected area */
+				if(is_effected(area, planes, co)) {
+					/* set or clear the hide flag */
+					BLI_BITMAP_MODIFY(gh, y * gridsize + x,
+									  action == PARTIALVIS_HIDE);
+
+					any_changed = 1;
+				}
+
+				/* keep track of whether any elements are still hidden */
+				if(BLI_BITMAP_GET(gh, y * gridsize + x))
+					any_hidden = 1;
+				else
+					any_visible = 1;
+			}
+		}
+
+		/* if everything in the grid is now visible, free the grid
+		   flags */
+		if(!any_hidden) {
+			MEM_freeN(gh);
+			grid_hidden[g] = NULL;
+		}
+	}
+
+	/* mark updates if anything was hidden/shown */
+	if(any_changed) {
+		BLI_pbvh_node_mark_rebuild_draw(node);
+		BLI_pbvh_node_fully_hidden_set(node, !any_visible);
+		multires_mark_as_modified(ob, MULTIRES_HIDDEN_MODIFIED);
+	}
+}
+
+static void rect_from_props(rcti *rect, PointerRNA *ptr)
+{
+	rect->xmin= RNA_int_get(ptr, "xmin");
+	rect->ymin= RNA_int_get(ptr, "ymin");
+	rect->xmax= RNA_int_get(ptr, "xmax");
+	rect->ymax= RNA_int_get(ptr, "ymax");
+}
+
+static void clip_planes_from_rect(bContext *C,
+								  float clip_planes[4][4],
+								  const rcti *rect)
+{
+	ViewContext vc;
+	BoundBox bb;
+	bglMats mats= {{0}};
+	
+	view3d_operator_needs_opengl(C);
+	view3d_set_viewcontext(C, &vc);
+	view3d_get_transformation(vc.ar, vc.rv3d, vc.obact, &mats);
+	ED_view3d_calc_clipping(&bb, clip_planes, &mats, rect);
+	mul_m4_fl(clip_planes, -1.0f);
+}
+
+/* If mode is inside, get all PBVH nodes that lie at least partially
+   inside the clip_planes volume. If mode is outside, get all nodes
+   that lie at least partially outside the volume. If showing all, get
+   all nodes. */
+static void get_pbvh_nodes(PBVH *pbvh,
+						   PBVHNode ***nodes,
+						   int *totnode,
+						   float clip_planes[4][4],
+						   PartialVisArea mode)
+{
+	BLI_pbvh_SearchCallback cb;
+
+	/* select search callback */
+	switch(mode) {
+	case PARTIALVIS_INSIDE:
+		cb = BLI_pbvh_node_planes_contain_AABB;
+		break;
+	case PARTIALVIS_OUTSIDE:
+		cb = BLI_pbvh_node_planes_exclude_AABB;
+		break;
+	case PARTIALVIS_ALL:
+		cb = NULL;
+	}
+	
+	BLI_pbvh_search_gather(pbvh, cb, clip_planes, nodes, totnode);
+}
+
+static int hide_show_exec(bContext *C, wmOperator *op)
+{
+	ARegion *ar = CTX_wm_region(C);
+	Object *ob = CTX_data_active_object(C);
+	Mesh *me = ob->data;
+	PartialVisAction action;
+	PartialVisArea area;
+	PBVH *pbvh;
+	PBVHNode **nodes;
+	DerivedMesh *dm;
+	PBVHType pbvh_type;
+	float clip_planes[4][4];
+	rcti rect;
+	int totnode, i;
+
+	/* read operator properties */
+	action = RNA_enum_get(op->ptr, "action");
+	area = RNA_enum_get(op->ptr, "area");
+	rect_from_props(&rect, op->ptr);
+
+	clip_planes_from_rect(C, clip_planes, &rect);
+
+	dm = mesh_get_derived_final(CTX_data_scene(C), ob, CD_MASK_BAREMESH);
+	pbvh = dm->getPBVH(ob, dm);
+	ob->sculpt->pbvh = pbvh;
+
+	get_pbvh_nodes(pbvh, &nodes, &totnode, clip_planes, area);
+	pbvh_type = BLI_pbvh_type(pbvh);
+
+	/* start undo */
+	switch(action) {
+	case PARTIALVIS_HIDE:
+		sculpt_undo_push_begin("Hide area");
+		break;
+	case PARTIALVIS_SHOW:
+		sculpt_undo_push_begin("Show area");
+		break;
+	}
+
+	for(i = 0; i < totnode; i++) {
+		switch(pbvh_type) {
+		case PBVH_FACES:
+			partialvis_update_mesh(ob, pbvh, nodes[i], action, area, clip_planes);
+			break;
+		case PBVH_GRIDS:
+			partialvis_update_grids(ob, pbvh, nodes[i], action, area, clip_planes);
+			break;
+		}
+	}
+
+	if(nodes)
+		MEM_freeN(nodes);
+	
+	/* end undo */
+	sculpt_undo_push_end();
+
+	/* ensure that edges and faces get hidden as well (not used by
+	   sculpt but it looks wrong when entering editmode otherwise) */
+	if(pbvh_type == PBVH_FACES) {
+		mesh_flush_hidden_from_verts(me->mvert, me->mloop,
+									 me->medge, me->totedge,
+									 me->mpoly, me->totpoly);
+	}
+
+	ED_region_tag_redraw(ar);
+	
+	return OPERATOR_FINISHED;
+}
+
+static int hide_show_invoke(bContext *C, wmOperator *op, wmEvent *event)
+{
+	PartialVisArea area = RNA_enum_get(op->ptr, "area");
+
+	if(area != PARTIALVIS_ALL)
+		return WM_border_select_invoke(C, op, event);
+	else
+		return op->type->exec(C, op);
+}
+
+void PAINT_OT_hide_show(struct wmOperatorType *ot)
+{
+	static EnumPropertyItem action_items[] = {
+		{PARTIALVIS_HIDE, "HIDE", 0, "Hide", "Hide vertices"},
+		{PARTIALVIS_SHOW, "SHOW", 0, "Show", "Show vertices"},
+		{0}};
+
+	static EnumPropertyItem area_items[] = {
+		{PARTIALVIS_OUTSIDE, "OUTSIDE", 0, "Outside",
+		 "Hide or show vertices outside the selection"},
+		{PARTIALVIS_INSIDE, "INSIDE", 0, "Inside",
+		 "Hide or show vertices inside the selection"},
+		{PARTIALVIS_ALL, "ALL", 0, "All",
+		 "Hide or show all vertices"},
+		{0}};
+	
+	/* identifiers */
+	ot->name = "Hide/Show";
+	ot->idname = "PAINT_OT_hide_show";
+
+	/* api callbacks */
+	ot->invoke = hide_show_invoke;
+	ot->modal = WM_border_select_modal;
+	ot->exec = hide_show_exec;
+	/* sculpt-only for now */
+	ot->poll = sculpt_mode_poll;
+
+	ot->flag = OPTYPE_REGISTER;
+
+	/* rna */
+	RNA_def_enum(ot->srna, "action", action_items, PARTIALVIS_HIDE,
+				 "Action", "Whether to hide or show vertices");
+	RNA_def_enum(ot->srna, "area", area_items, PARTIALVIS_INSIDE,
+				 "Area", "Which vertices to hide or show");
+	
+	RNA_def_int(ot->srna, "xmin", 0, INT_MIN, INT_MAX, "X Min", "", INT_MIN, INT_MAX);
+	RNA_def_int(ot->srna, "xmax", 0, INT_MIN, INT_MAX, "X Max", "", INT_MIN, INT_MAX);
+	RNA_def_int(ot->srna, "ymin", 0, INT_MIN, INT_MAX, "Y Min", "", INT_MIN, INT_MAX);
+	RNA_def_int(ot->srna, "ymax", 0, INT_MIN, INT_MAX, "Y Max", "", INT_MIN, INT_MAX);
+}

source/blender/editors/sculpt_paint/paint_intern.h

@@ -168,5 +168,19 @@ struct ListBase *undo_paint_push_get_list(int type);
 void undo_paint_push_count_alloc(int type, int size);
 void undo_paint_push_end(int type);
 
-#endif /* __PAINT_INTERN_H__ */
+/* paint_hide.c */
 
+typedef enum {
+	PARTIALVIS_HIDE,
+	PARTIALVIS_SHOW
+} PartialVisAction;
+
+typedef enum {
+	PARTIALVIS_INSIDE,
+	PARTIALVIS_OUTSIDE,
+	PARTIALVIS_ALL
+} PartialVisArea;
+
+void PAINT_OT_hide_show(struct wmOperatorType *ot);
+
+#endif /* __PAINT_INTERN_H__ */

source/blender/editors/sculpt_paint/paint_ops.c

@@ -426,6 +426,9 @@ void ED_operatortypes_paint(void)
 	WM_operatortype_append(PAINT_OT_face_select_inverse);
 	WM_operatortype_append(PAINT_OT_face_select_hide);
 	WM_operatortype_append(PAINT_OT_face_select_reveal);
+
+	/* partial visibility */
+	WM_operatortype_append(PAINT_OT_hide_show);
 }
 
 
@@ -520,6 +523,22 @@ static void ed_keymap_paint_brush_radial_control(wmKeyMap *keymap, const char *p
 	}
 }
 
+void paint_partial_visibility_keys(wmKeyMap *keymap)
+{
+	wmKeyMapItem *kmi;
+	
+	/* Partial visiblity */
+	kmi= WM_keymap_add_item(keymap, "PAINT_OT_hide_show", HKEY, KM_PRESS, KM_CTRL, 0);
+	RNA_enum_set(kmi->ptr, "action", PARTIALVIS_SHOW);
+	RNA_enum_set(kmi->ptr, "area", PARTIALVIS_INSIDE);
+	kmi= WM_keymap_add_item(keymap, "PAINT_OT_hide_show", HKEY, KM_PRESS, 0, 0);
+	RNA_enum_set(kmi->ptr, "action", PARTIALVIS_HIDE);
+	RNA_enum_set(kmi->ptr, "area", PARTIALVIS_INSIDE);
+	kmi= WM_keymap_add_item(keymap, "PAINT_OT_hide_show", HKEY, KM_PRESS, KM_ALT, 0);
+	RNA_enum_set(kmi->ptr, "action", PARTIALVIS_SHOW);
+	RNA_enum_set(kmi->ptr, "area", PARTIALVIS_ALL);
+}
+
 void ED_keymap_paint(wmKeyConfig *keyconf)
 {
 	wmKeyMap *keymap;
@@ -534,6 +553,9 @@ void ED_keymap_paint(wmKeyConfig *keyconf)
 	RNA_enum_set(WM_keymap_add_item(keymap, "SCULPT_OT_brush_stroke", LEFTMOUSE, KM_PRESS, KM_CTRL,  0)->ptr, "mode", BRUSH_STROKE_INVERT);
 	RNA_enum_set(WM_keymap_add_item(keymap, "SCULPT_OT_brush_stroke", LEFTMOUSE, KM_PRESS, KM_SHIFT, 0)->ptr, "mode", BRUSH_STROKE_SMOOTH);
 
+	/* Partial visibility, sculpt-only for now */
+	paint_partial_visibility_keys(keymap);
+
 	for(i=0; i<=5; i++)
 		RNA_int_set(WM_keymap_add_item(keymap, "OBJECT_OT_subdivision_set", ZEROKEY+i, KM_PRESS, KM_CTRL, 0)->ptr, "level", i);
 

source/blender/editors/sculpt_paint/sculpt.c

@@ -805,7 +805,7 @@ static void calc_area_normal(Sculpt *sd, Object *ob, float an[3], PBVHNode **nod
 		float private_an[3] = {0.0f, 0.0f, 0.0f};
 		float private_out_flip[3] = {0.0f, 0.0f, 0.0f};
 
-		unode = sculpt_undo_push_node(ob, nodes[n]);
+		unode = sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		sculpt_brush_test_init(ss, &test);
 
 		if(ss->cache->original) {
@@ -1295,7 +1295,7 @@ static void do_grab_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode)
 		short (*origno)[3];
 		float (*proxy)[3];
 
-		unode=  sculpt_undo_push_node(ob, nodes[n]);
+		unode=  sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		origco= unode->co;
 		origno= unode->no;
 
@@ -1436,7 +1436,7 @@ static void do_thumb_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode
 		short (*origno)[3];
 		float (*proxy)[3];
 
-		unode=  sculpt_undo_push_node(ob, nodes[n]);
+		unode=  sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		origco= unode->co;
 		origno= unode->no;
 
@@ -1490,7 +1490,7 @@ static void do_rotate_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnod
 		short (*origno)[3];
 		float (*proxy)[3];
 
-		unode=  sculpt_undo_push_node(ob, nodes[n]);
+		unode=  sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		origco= unode->co;
 		origno= unode->no;
 
@@ -1541,7 +1541,7 @@ static void do_layer_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode
 
 		//proxy= BLI_pbvh_node_add_proxy(ss->pbvh, nodes[n])->co;
 		
-		unode= sculpt_undo_push_node(ob, nodes[n]);
+		unode= sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		origco=unode->co;
 		if (!unode->layer_disp) {
 			#pragma omp critical 
@@ -1643,7 +1643,7 @@ static void calc_flatten_center(Sculpt *sd, Object *ob, PBVHNode **nodes, int to
 		float private_fc[3] = {0.0f, 0.0f, 0.0f};
 		int private_count = 0;
 
-		unode = sculpt_undo_push_node(ob, nodes[n]);
+		unode = sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		sculpt_brush_test_init(ss, &test);
 
 		if(ss->cache->original) {
@@ -1708,7 +1708,7 @@ static void calc_area_normal_and_flatten_center(Sculpt *sd, Object *ob,
 		float private_fc[3] = {0.0f, 0.0f, 0.0f};
 		int private_count = 0;
 
-		unode = sculpt_undo_push_node(ob, nodes[n]);
+		unode = sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 		sculpt_brush_test_init(ss, &test);
 
 		if(ss->cache->original) {
@@ -2319,7 +2319,7 @@ static void do_brush_action(Sculpt *sd, Object *ob, Brush *brush)
 	if (totnode) {
 		#pragma omp parallel for schedule(guided) if (sd->flags & SCULPT_USE_OPENMP)
 		for (n= 0; n < totnode; n++) {
-			sculpt_undo_push_node(ob, nodes[n]);
+			sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS);
 			BLI_pbvh_node_mark_update(nodes[n]);
 		}
 
@@ -2432,7 +2432,7 @@ static void sculpt_combine_proxies(Sculpt *sd, Object *ob)
 			float (*orco)[3];
 
 			if(use_orco)
-				orco= sculpt_undo_push_node(ob, nodes[n])->co;
+				orco= sculpt_undo_push_node(ob, nodes[n], SCULPT_UNDO_COORDS)->co;
 
 			BLI_pbvh_node_get_proxies(nodes[n], &proxies, &proxy_count);
 
@@ -2737,7 +2737,7 @@ void sculpt_update_mesh_elements(Scene *scene, Sculpt *sd, Object *ob, int need_
 	}
 }
 
-static int sculpt_mode_poll(bContext *C)
+int sculpt_mode_poll(bContext *C)
 {
 	Object *ob = CTX_data_active_object(C);
 	return ob && ob->mode & OB_MODE_SCULPT;

source/blender/editors/sculpt_paint/sculpt_intern.h

@@ -37,6 +37,7 @@
 #include "DNA_vec_types.h"
 #include "DNA_key_types.h"
 
+#include "BLI_bitmap.h"
 #include "BLI_pbvh.h"
 
 struct bContext;
@@ -54,6 +55,7 @@ struct MultiresModifierData *sculpt_multires_active(struct Scene *scene, struct
 
 void sculpt(Sculpt *sd);
 
+int sculpt_mode_poll(struct bContext *C);
 int sculpt_poll(struct bContext *C);
 void sculpt_update_mesh_elements(struct Scene *scene, struct Sculpt *sd, struct Object *ob, int need_pmap);
 
@@ -65,9 +67,16 @@ int sculpt_stroke_get_location(bContext *C, float out[3], float mouse[2]);
 
 /* Undo */
 
+typedef enum {
+	SCULPT_UNDO_COORDS,
+	SCULPT_UNDO_HIDDEN
+} SculptUndoType;
+
 typedef struct SculptUndoNode {
 	struct SculptUndoNode *next, *prev;
 
+	SculptUndoType type;
+
 	char idname[MAX_ID_NAME];	/* name instead of pointer*/
 	void *node;					/* only during push, not valid afterwards! */
 
@@ -79,12 +88,14 @@ typedef struct SculptUndoNode {
 	/* non-multires */
 	int maxvert;				/* to verify if totvert it still the same */
 	int *index;					/* to restore into right location */
+	BLI_bitmap vert_hidden;
 
 	/* multires */
 	int maxgrid;				/* same for grid */
 	int gridsize;				/* same for grid */
 	int totgrid;				/* to restore into right location */
 	int *grids;					/* to restore into right location */
+	BLI_bitmap *grid_hidden;
 
 	/* layer brush */
 	float *layer_disp;
@@ -93,7 +104,7 @@ typedef struct SculptUndoNode {
 	char shapeName[sizeof(((KeyBlock *)0))->name];
 } SculptUndoNode;
 
-SculptUndoNode *sculpt_undo_push_node(Object *ob, PBVHNode *node);
+SculptUndoNode *sculpt_undo_push_node(Object *ob, PBVHNode *node, SculptUndoType type);
 SculptUndoNode *sculpt_undo_get_node(PBVHNode *node);
 void sculpt_undo_push_begin(const char *name);
 void sculpt_undo_push_end(void);

source/blender/editors/sculpt_paint/sculpt_undo.c

@@ -68,9 +68,12 @@
 
 /************************** Undo *************************/
 
-static void update_cb(PBVHNode *node, void *UNUSED(unused))
+static void update_cb(PBVHNode *node, void *rebuild)
 {
 	BLI_pbvh_node_mark_update(node);
+	if(*((int*)rebuild))
+		BLI_pbvh_node_mark_rebuild_draw(node);
+	BLI_pbvh_node_fully_hidden_set(node, 0);
 }
 
 static void sculpt_undo_restore_deformed(SculptSession *ss, SculptUndoNode *unode, int uindex, int oindex, float coord[3])
@@ -165,6 +168,44 @@ static int sculpt_undo_restore_coords(bContext *C, DerivedMesh *dm, SculptUndoNo
 	return 1;
 }
 
+static int sculpt_undo_restore_hidden(bContext *C, DerivedMesh *dm,
+									  SculptUndoNode *unode)
+{
+	Object *ob = CTX_data_active_object(C);
+	SculptSession *ss = ob->sculpt;
+	int i;
+
+	if(unode->maxvert) {
+		MVert *mvert= ss->mvert;
+		
+		for(i=0; i<unode->totvert; i++) {
+			MVert *v = &mvert[unode->index[i]];
+			int uval= BLI_BITMAP_GET(unode->vert_hidden, i);
+
+			BLI_BITMAP_MODIFY(unode->vert_hidden, i,
+							  v->flag & ME_HIDE);
+			if(uval)
+				v->flag |= ME_HIDE;
+			else
+				v->flag &= ~ME_HIDE;
+			
+			v->flag |= ME_VERT_PBVH_UPDATE;
+		}
+	}
+	else if(unode->maxgrid && dm->getGridData) {
+		BLI_bitmap *grid_hidden = dm->getGridHidden(dm);
+		
+		for(i=0; i<unode->totgrid; i++) {
+			SWAP(BLI_bitmap,
+				 unode->grid_hidden[i],
+				 grid_hidden[unode->grids[i]]);
+			
+		}
+	}
+
+	return 1;
+}
+
 static void sculpt_undo_restore(bContext *C, ListBase *lb)
 {
 	Scene *scene = CTX_data_scene(C);
@@ -174,7 +215,7 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 	SculptSession *ss = ob->sculpt;
 	SculptUndoNode *unode;
 	MultiresModifierData *mmd;
-	int update= 0;
+	int update= 0, rebuild= 1;
 
 	sculpt_update_mesh_elements(scene, sd, ob, 0);
 
@@ -197,20 +238,32 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 			continue;
 		}
 
-		if(sculpt_undo_restore_coords(C, dm, unode))
-			update= 1;
+		switch(unode->type) {
+		case SCULPT_UNDO_COORDS:
+			if(sculpt_undo_restore_coords(C, dm, unode))
+				update= 1;
+			break;
+		case SCULPT_UNDO_HIDDEN:
+			if(sculpt_undo_restore_hidden(C, dm, unode))
+				rebuild= 1;
+			break;
+		}
 	}
 
-	if(update) {
+	if(update || rebuild) {
 		int tag_update= 0;
 		/* we update all nodes still, should be more clever, but also
 		 * needs to work correct when exiting/entering sculpt mode and
 		 * the nodes get recreated, though in that case it could do all */
-		BLI_pbvh_search_callback(ss->pbvh, NULL, NULL, update_cb, NULL);
+		BLI_pbvh_search_callback(ss->pbvh, NULL, NULL, update_cb, &rebuild);
 		BLI_pbvh_update(ss->pbvh, PBVH_UpdateBB|PBVH_UpdateOriginalBB|PBVH_UpdateRedraw, NULL);
 
-		if((mmd=sculpt_multires_active(scene, ob)))
-			multires_mark_as_modified(ob, MULTIRES_COORDS_MODIFIED);
+		if((mmd=sculpt_multires_active(scene, ob))) {
+			if(rebuild)
+				multires_mark_as_modified(ob, MULTIRES_HIDDEN_MODIFIED);
+			else
+				multires_mark_as_modified(ob, MULTIRES_COORDS_MODIFIED);
+		}
 
 		tag_update= ((Mesh*)ob->data)->id.us > 1;
 
@@ -233,6 +286,7 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 static void sculpt_undo_free(ListBase *lb)
 {
 	SculptUndoNode *unode;
+	int i;
 
 	for(unode=lb->first; unode; unode=unode->next) {
 		if(unode->co)
@@ -247,6 +301,15 @@ static void sculpt_undo_free(ListBase *lb)
 			MEM_freeN(unode->layer_disp);
 		if(unode->orig_co)
 			MEM_freeN(unode->orig_co);
+		if(unode->vert_hidden)
+			MEM_freeN(unode->vert_hidden);
+		if(unode->grid_hidden) {
+			for(i=0; i<unode->totgrid; i++) {
+				if(unode->grid_hidden[i])
+					MEM_freeN(unode->grid_hidden[i]);
+			}
+			MEM_freeN(unode->grid_hidden);
+		}
 	}
 }
 
@@ -265,7 +328,31 @@ SculptUndoNode *sculpt_undo_get_node(PBVHNode *node)
 	return NULL;
 }
 
-SculptUndoNode *sculpt_undo_alloc_node(Object *ob, PBVHNode *node)
+static void sculpt_undo_alloc_and_store_hidden(PBVH *pbvh,
+											   SculptUndoNode *unode)
+{
+	PBVHNode *node= unode->node;
+	BLI_bitmap *grid_hidden;
+	int i, *grid_indices, totgrid;
+
+	grid_hidden= BLI_pbvh_grid_hidden(pbvh);
+
+	BLI_pbvh_node_get_grids(pbvh, node, &grid_indices, &totgrid,
+							NULL, NULL, NULL, NULL);
+			
+	unode->grid_hidden= MEM_mapallocN(sizeof(BLI_bitmap) * totgrid,
+									  "unode->grid_hidden");
+		
+	for(i = 0; i < totgrid; i++) {
+		if(grid_hidden[i])
+			unode->grid_hidden[i] = MEM_dupallocN(grid_hidden[i]);
+		else
+			unode->grid_hidden[i] = NULL;
+	}
+}
+
+static SculptUndoNode *sculpt_undo_alloc_node(Object *ob, PBVHNode *node,
+											  SculptUndoType type)
 {
 	ListBase *lb= undo_paint_push_get_list(UNDO_PAINT_MESH);
 	SculptUndoNode *unode;
@@ -274,6 +361,7 @@ SculptUndoNode *sculpt_undo_alloc_node(Object *ob, PBVHNode *node)
 	
 	unode= MEM_callocN(sizeof(SculptUndoNode), "SculptUndoNode");
 	BLI_strncpy(unode->idname, ob->id.name, sizeof(unode->idname));
+	unode->type= type;
 	unode->node= node;
 
 	BLI_pbvh_node_num_verts(ss->pbvh, node, &totvert, &allvert);
@@ -281,10 +369,25 @@ SculptUndoNode *sculpt_undo_alloc_node(Object *ob, PBVHNode *node)
 		&maxgrid, &gridsize, NULL, NULL);
 
 	unode->totvert= totvert;
+	
 	/* we will use this while sculpting, is mapalloc slow to access then? */
-	unode->co= MEM_mapallocN(sizeof(float)*3*allvert, "SculptUndoNode.co");
-	unode->no= MEM_mapallocN(sizeof(short)*3*allvert, "SculptUndoNode.no");
-	undo_paint_push_count_alloc(UNDO_PAINT_MESH, (sizeof(float)*3 + sizeof(short)*3 + sizeof(int))*allvert);
+
+	/* general TODO, fix count_alloc */
+	switch(type) {
+	case SCULPT_UNDO_COORDS:
+		unode->co= MEM_mapallocN(sizeof(float)*3*allvert, "SculptUndoNode.co");
+		unode->no= MEM_mapallocN(sizeof(short)*3*allvert, "SculptUndoNode.no");
+		undo_paint_push_count_alloc(UNDO_PAINT_MESH, (sizeof(float)*3 + sizeof(short)*3 + sizeof(int))*allvert);
+		break;
+	case SCULPT_UNDO_HIDDEN:
+		if(maxgrid)
+			sculpt_undo_alloc_and_store_hidden(ss->pbvh, unode);
+		else
+			unode->vert_hidden= BLI_BITMAP_NEW(allvert, "SculptUndoNode.vert_hidden");
+		
+		break;
+	}
+	
 	BLI_addtail(lb, unode);
 
 	if(maxgrid) {
@@ -306,7 +409,46 @@ SculptUndoNode *sculpt_undo_alloc_node(Object *ob, PBVHNode *node)
 	return unode;
 }
 
-SculptUndoNode *sculpt_undo_push_node(Object *ob, PBVHNode *node)
+static void sculpt_undo_store_coords(Object *ob, SculptUndoNode *unode)
+{
+	SculptSession *ss = ob->sculpt;
+	PBVHVertexIter vd;
+
+	BLI_pbvh_vertex_iter_begin(ss->pbvh, unode->node, vd, PBVH_ITER_ALL) {
+		copy_v3_v3(unode->co[vd.i], vd.co);
+		if(vd.no) copy_v3_v3_short(unode->no[vd.i], vd.no);
+		else normal_float_to_short_v3(unode->no[vd.i], vd.fno);
+
+		if(ss->modifiers_active)
+			copy_v3_v3(unode->orig_co[vd.i], ss->orig_cos[unode->index[vd.i]]);
+	}
+	BLI_pbvh_vertex_iter_end;
+}
+
+static void sculpt_undo_store_hidden(Object *ob, SculptUndoNode *unode)
+{
+	PBVH *pbvh= ob->sculpt->pbvh;
+	PBVHNode *node= unode->node;
+
+	if(unode->grids) {
+		/* already stored during allocation */
+ 	}
+	else {
+		MVert *mvert;
+		int *vert_indices, allvert;
+		int i;
+		
+		BLI_pbvh_node_num_verts(pbvh, node, NULL, &allvert);
+		BLI_pbvh_node_get_verts(pbvh, node, &vert_indices, &mvert);
+		for(i = 0; i < allvert; i++) {
+			BLI_BITMAP_MODIFY(unode->vert_hidden, i,
+							  mvert[vert_indices[i]].flag & ME_HIDE);
+		}
+	}
+}
+
+SculptUndoNode *sculpt_undo_push_node(Object *ob, PBVHNode *node,
+									  SculptUndoType type)
 {
 	SculptSession *ss = ob->sculpt;
 	SculptUndoNode *unode;
@@ -319,24 +461,18 @@ SculptUndoNode *sculpt_undo_push_node(Object *ob, PBVHNode *node)
 		return unode;
 	}
 
-	unode= sculpt_undo_alloc_node(ob, node);
-
+	unode= sculpt_undo_alloc_node(ob, node, type);
+	
 	BLI_unlock_thread(LOCK_CUSTOM1);
 
 	/* copy threaded, hopefully this is the performance critical part */
-	{
-		PBVHVertexIter vd;
-
-		BLI_pbvh_vertex_iter_begin(ss->pbvh, node, vd, PBVH_ITER_ALL) {
-			copy_v3_v3(unode->co[vd.i], vd.co);
-			if(vd.no) copy_v3_v3_short(unode->no[vd.i], vd.no);
-			else normal_float_to_short_v3(unode->no[vd.i], vd.fno);
-			if(vd.vert_indices) unode->index[vd.i]= vd.vert_indices[vd.i];
-
-			if(ss->modifiers_active)
-				copy_v3_v3(unode->orig_co[vd.i], ss->orig_cos[unode->index[vd.i]]);
-		}
-		BLI_pbvh_vertex_iter_end;
+	switch(type) {
+	case SCULPT_UNDO_COORDS:
+		sculpt_undo_store_coords(ob, unode);
+		break;
+	case SCULPT_UNDO_HIDDEN:
+		sculpt_undo_store_hidden(ob, unode);
+		break;
 	}
 
 	if(unode->grids) {
@@ -345,6 +481,12 @@ SculptUndoNode *sculpt_undo_push_node(Object *ob, PBVHNode *node)
 								NULL, NULL, NULL, NULL);
 		memcpy(unode->grids, grids, sizeof(int)*totgrid);
 	}
+	else {
+		int *vert_indices, allvert;
+		BLI_pbvh_node_num_verts(ss->pbvh, node, NULL, &allvert);
+		BLI_pbvh_node_get_verts(ss->pbvh, node, &vert_indices, NULL);
+		memcpy(unode->index, vert_indices, sizeof(int)*unode->totvert);
+	}
 
 	/* store active shape key */
 	if(ss->kb) BLI_strncpy(unode->shapeName, ss->kb->name, sizeof(ss->kb->name));

source/blender/windowmanager/intern/wm_operators.c

@@ -3831,6 +3831,7 @@ static void gesture_border_modal_keymap(wmKeyConfig *keyconf)
 	WM_modalkeymap_assign(keymap, "MARKER_OT_select_border");
 	WM_modalkeymap_assign(keymap, "NLA_OT_select_border");
 	WM_modalkeymap_assign(keymap, "NODE_OT_select_border");
+	WM_modalkeymap_assign(keymap, "PAINT_OT_hide_show");
 	WM_modalkeymap_assign(keymap, "OUTLINER_OT_select_border");
 //	WM_modalkeymap_assign(keymap, "SCREEN_OT_border_select"); // template
 	WM_modalkeymap_assign(keymap, "SEQUENCER_OT_select_border");