Fix T39517,

Issue here is that "show diffuse" option does not respect its intended
purpose which is to be used only for masking.

There are a couple of caveats here:

Dyntopo and multires -always- have mask data enabled, and thus as soon
as one goes to dyntopo mode or adds a multires modifier he would get the
default grey color instead.

Matcaps would break when nodes asked for a diffuse material color (this
was broken before too). Solved by adding global material state for when
matcaps are enabled. Also matcaps don't always played well with VBOs
off.

Added a few more missing updates for mask operators to notify
show_diffuse property as changed. This was also needed on rebuilding
dyntopo pbvh.

Also make zero mask color duller again after artist feedback.

source/blender/blenkernel/intern/cdderivedmesh.c

@@ -270,6 +270,8 @@ static PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 		BKE_pbvh_build_bmesh(cddm->pbvh, ob->sculpt->bm,
 		                     ob->sculpt->bm_smooth_shading,
 		                     ob->sculpt->bm_log);
+
+		pbvh_show_diffuse_color_set(cddm->pbvh, ob->sculpt->show_diffuse_color);
 	}
 		
 

source/blender/blenkernel/intern/pbvh.c

@@ -1925,5 +1925,20 @@ void pbvh_vertex_iter_init(PBVH *bvh, PBVHNode *node,
 
 void pbvh_show_diffuse_color_set(PBVH *bvh, bool show_diffuse_color)
 {
-	bvh->show_diffuse_color = show_diffuse_color;
+	bool has_mask = false;
+
+	switch (bvh->type) {
+		case PBVH_GRIDS:
+			has_mask = (bvh->gridkey.has_mask != 0);
+			break;
+		case PBVH_FACES:
+			has_mask = (bvh->vdata && CustomData_get_layer(bvh->vdata,
+			                                CD_PAINT_MASK));
+			break;
+		case PBVH_BMESH:
+			has_mask = (bvh->bm && (CustomData_get_offset(&bvh->bm->vdata, CD_PAINT_MASK) != -1));
+			break;
+	}
+
+	bvh->show_diffuse_color = !has_mask || show_diffuse_color;
 }

source/blender/editors/sculpt_paint/paint_mask.c

@@ -94,9 +94,7 @@ static int mask_flood_fill_exec(bContext *C, wmOperator *op)
 	PBVH *pbvh;
 	PBVHNode **nodes;
 	int totnode, i;
-#ifdef _OPENMP
 	Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
-#endif
 
 	mode = RNA_enum_get(op->ptr, "mode");
 	value = RNA_float_get(op->ptr, "value");
@@ -107,6 +105,9 @@ static int mask_flood_fill_exec(bContext *C, wmOperator *op)
 	pbvh = dm->getPBVH(ob, dm);
 	ob->sculpt->pbvh = pbvh;
 
+	ob->sculpt->show_diffuse_color = sd->flags & SCULPT_SHOW_DIFFUSE;
+	pbvh_show_diffuse_color_set(pbvh, ob->sculpt->show_diffuse_color);
+
 	BKE_pbvh_search_gather(pbvh, NULL, NULL, &nodes, &totnode);
 
 	sculpt_undo_push_begin("Mask flood fill");
@@ -218,6 +219,9 @@ int do_sculpt_mask_box_select(ViewContext *vc, rcti *rect, bool select, bool UNU
 	pbvh = dm->getPBVH(ob, dm);
 	ob->sculpt->pbvh = pbvh;
 
+	ob->sculpt->show_diffuse_color = sd->flags & SCULPT_SHOW_DIFFUSE;
+	pbvh_show_diffuse_color_set(pbvh, ob->sculpt->show_diffuse_color);
+
 	sculpt_undo_push_begin("Mask box fill");
 
 	for (symmpass = 0; symmpass <= symm; ++symmpass) {
@@ -362,6 +366,9 @@ static int paint_mask_gesture_lasso_exec(bContext *C, wmOperator *op)
 		pbvh = dm->getPBVH(ob, dm);
 		ob->sculpt->pbvh = pbvh;
 
+		ob->sculpt->show_diffuse_color = sd->flags & SCULPT_SHOW_DIFFUSE;
+		pbvh_show_diffuse_color_set(pbvh, ob->sculpt->show_diffuse_color);
+
 		sculpt_undo_push_begin("Mask lasso fill");
 
 		for (symmpass = 0; symmpass <= symm; ++symmpass) {

source/blender/gpu/GPU_buffers.h

@@ -180,7 +180,7 @@ void GPU_update_bmesh_pbvh_buffers(GPU_PBVH_Buffers *buffers,
                               struct GHash *bm_faces,
                               struct GSet *bm_unique_verts,
                               struct GSet *bm_other_verts,
-                              int show_diffuse_color);
+                              bool show_diffuse_color);
 
 void GPU_update_grid_pbvh_buffers(GPU_PBVH_Buffers *buffers, struct CCGElem **grids,
                              const struct DMFlagMat *grid_flag_mats,

source/blender/gpu/GPU_draw.h

@@ -76,6 +76,7 @@ int GPU_enable_material(int nr, void *attribs);
 void GPU_disable_material(void);
 
 void GPU_material_diffuse_get(int nr, float diff[4]);
+bool GPU_material_use_matcaps_get(void);
 
 void GPU_set_material_alpha_blend(int alphablend);
 int GPU_get_material_alpha_blend(void);

source/blender/gpu/intern/gpu_buffers.c

@@ -1354,6 +1354,7 @@ struct GPU_PBVH_Buffers {
 	int smooth;
 
 	bool show_diffuse_color;
+	bool use_matcaps;
 	float diffuse_color[4];
 };
 typedef enum {
@@ -1443,12 +1444,15 @@ void GPU_update_mesh_pbvh_buffers(GPU_PBVH_Buffers *buffers, MVert *mvert,
 
 	buffers->vmask = vmask;
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	if (buffers->vert_buf) {
 		int totelem = (buffers->smooth ? totvert : (buffers->tot_tri * 3));
-		float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+		float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 0.8f};
 
-		if (buffers->show_diffuse_color) {
+		if (buffers->use_matcaps)
+			diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+		else if (show_diffuse_color) {
 			MFace *f = buffers->mface + buffers->face_indices[0];
 
 			GPU_material_diffuse_get(f->mat_nr + 1, diffuse_color);
@@ -1583,6 +1587,7 @@ GPU_PBVH_Buffers *GPU_build_pbvh_mesh_buffers(int (*face_vert_indices)[4],
 	buffers->smooth = mface[face_indices[0]].flag & ME_SMOOTH;
 
 	buffers->show_diffuse_color = false;
+	buffers->use_matcaps = false;
 
 	/* Count the number of visible triangles */
 	for (i = 0, tottri = 0; i < totface; ++i) {
@@ -1658,15 +1663,18 @@ void GPU_update_grid_pbvh_buffers(GPU_PBVH_Buffers *buffers, CCGElem **grids,
 	int i, j, k, x, y;
 
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	/* Build VBO */
 	if (buffers->vert_buf) {
 		int totvert = key->grid_area * totgrid;
 		int smooth = grid_flag_mats[grid_indices[0]].flag & ME_SMOOTH;
 		const int has_mask = key->has_mask;
-		float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+		float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
-		if (buffers->show_diffuse_color) {
+		if (buffers->use_matcaps)
+			diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+		else if (show_diffuse_color) {
 			const DMFlagMat *flags = &grid_flag_mats[grid_indices[0]];
 
 			GPU_material_diffuse_get(flags->mat_nr + 1, diffuse_color);
@@ -1903,6 +1911,7 @@ GPU_PBVH_Buffers *GPU_build_grid_pbvh_buffers(int *grid_indices, int totgrid,
 	buffers->totgrid = totgrid;
 
 	buffers->show_diffuse_color = false;
+	buffers->use_matcaps = false;
 
 	/* Count the number of quads */
 	totquad = gpu_count_grid_quads(grid_hidden, grid_indices, totgrid, gridsize);
@@ -2023,17 +2032,18 @@ void GPU_update_bmesh_pbvh_buffers(GPU_PBVH_Buffers *buffers,
                               GHash *bm_faces,
                               GSet *bm_unique_verts,
                               GSet *bm_other_verts,
-                              int show_diffuse_color)
+                              bool show_diffuse_color)
 {
 	VertexBufferFormat *vert_data;
 	void *tri_data;
 	int tottri, totvert, maxvert = 0;
-	float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
 	/* TODO, make mask layer optional for bmesh buffer */
 	const int cd_vert_mask_offset = CustomData_get_offset(&bm->vdata, CD_PAINT_MASK);
 
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	if (!buffers->vert_buf || (buffers->smooth && !buffers->index_buf))
 		return;
@@ -2061,8 +2071,10 @@ void GPU_update_bmesh_pbvh_buffers(GPU_PBVH_Buffers *buffers,
 		return;
 	}
 
-	if (show_diffuse_color) {
+	if (buffers->use_matcaps)
-		/* due to dynamc nature of dyntopo, only get first material */
+		diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+	else if (show_diffuse_color) {
+		/* due to dynamic nature of dyntopo, only get first material */
 		GHashIterator gh_iter;
 		BMFace *f;
 		BLI_ghashIterator_init(&gh_iter, bm_faces);
@@ -2215,6 +2227,7 @@ GPU_PBVH_Buffers *GPU_build_bmesh_pbvh_buffers(int smooth_shading)
 	buffers->use_bmesh = true;
 	buffers->smooth = smooth_shading;
 	buffers->show_diffuse_color = false;
+	buffers->use_matcaps = false;
 
 	return buffers;
 }
@@ -2225,9 +2238,11 @@ static void gpu_draw_buffers_legacy_mesh(GPU_PBVH_Buffers *buffers)
 	int i, j;
 	const int has_mask = (buffers->vmask != NULL);
 	const MFace *face = &buffers->mface[buffers->face_indices[0]];
-	float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
-	if (buffers->show_diffuse_color)
+	if (buffers->use_matcaps)
+		diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+	else if (buffers->show_diffuse_color)
 		GPU_material_diffuse_get(face->mat_nr + 1, diffuse_color);
 
 	if (has_mask) {
@@ -2297,9 +2312,11 @@ static void gpu_draw_buffers_legacy_grids(GPU_PBVH_Buffers *buffers)
 	int i, j, x, y, gridsize = buffers->gridkey.grid_size;
 	const int has_mask = key->has_mask;
 	const DMFlagMat *flags = &buffers->grid_flag_mats[buffers->grid_indices[0]];
-	float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
-	if (buffers->show_diffuse_color)
+	if (buffers->use_matcaps)
+		diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+	else if (buffers->show_diffuse_color)
 		GPU_material_diffuse_get(flags->mat_nr + 1, diffuse_color);
 
 	if (has_mask) {
@@ -2517,6 +2534,9 @@ bool GPU_pbvh_buffers_diffuse_changed(GPU_PBVH_Buffers *buffers, GHash *bm_faces
 	if (buffers->show_diffuse_color != show_diffuse_color)
 		return true;
 
+	if (buffers->use_matcaps != GPU_material_use_matcaps_get())
+		return true;
+
 	if (buffers->show_diffuse_color == false)
 		return false;
 

source/blender/gpu/intern/gpu_draw.c

@@ -1342,6 +1342,7 @@ static struct GPUMaterialState {
 	GPUBlendMode *alphablend;
 	GPUBlendMode alphablend_fixed[FIXEDMAT];
 	bool use_alpha_pass, is_alpha_pass;
+	bool use_matcaps;
 
 	int lastmatnr, lastretval;
 	GPUBlendMode lastalphablend;
@@ -1409,6 +1410,7 @@ void GPU_begin_object_materials(View3D *v3d, RegionView3D *rv3d, Scene *scene, O
 	GMS.lastmatnr = -1;
 	GMS.lastretval = -1;
 	GMS.lastalphablend = GPU_BLEND_SOLID;
+	GMS.use_matcaps = use_matcap;
 
 	GMS.backface_culling = (v3d->flag2 & V3D_BACKFACE_CULLING) != 0;
 
@@ -1656,6 +1658,12 @@ void GPU_material_diffuse_get(int nr, float diff[4])
 	}
 }
 
+bool GPU_material_use_matcaps_get(void)
+{
+	return GMS.use_matcaps;
+}
+
+
 void GPU_end_object_materials(void)
 {
 	GPU_disable_material();