Cycles: add per layer collection mask/holdout support.

In the outliner, right click > view layer > set holdout. This is
temporary until we have more general dynamic overrides, but helps
Spring production for now.

intern/cycles/blender/blender_object.cpp

@@ -291,6 +291,7 @@ void BlenderSync::sync_background_light(bool use_portal)
 /* Object */
 
 Object *BlenderSync::sync_object(BL::Depsgraph& b_depsgraph,
+                                 BL::ViewLayer& b_view_layer,
                                  BL::DepsgraphObjectInstance& b_instance,
                                  float motion_time,
                                  bool hide_tris,
@@ -345,7 +346,8 @@ Object *BlenderSync::sync_object(BL::Depsgraph& b_depsgraph,
 
 	/* Visibility flags for both parent and child. */
 	PointerRNA cobject = RNA_pointer_get(&b_ob.ptr, "cycles");
-	bool use_holdout = get_boolean(cobject, "is_holdout");
+	bool use_holdout = get_boolean(cobject, "is_holdout") ||
+	                   b_ob.holdout_get(b_view_layer);
 	uint visibility = object_ray_visibility(b_ob) & PATH_RAY_ALL_VISIBILITY;
 
 	if(b_parent.ptr.data != b_ob.ptr.data) {
@@ -588,6 +590,7 @@ void BlenderSync::sync_objects(BL::Depsgraph& b_depsgraph, float motion_time)
 	bool cancel = false;
 	bool use_portal = false;
 
+	BL::ViewLayer b_view_layer = b_depsgraph.view_layer_eval();
 	BL::Depsgraph::mode_enum depsgraph_mode = b_depsgraph.mode();
 
 	BL::Depsgraph::object_instances_iterator b_instance_iter;
@@ -609,6 +612,7 @@ void BlenderSync::sync_objects(BL::Depsgraph& b_depsgraph, float motion_time)
 		 if(!object_render_hide(b_ob, true, true, hide_tris, depsgraph_mode)) {
 			/* object itself */
 			sync_object(b_depsgraph,
+			            b_view_layer,
 			            b_instance,
 			            motion_time,
 			            hide_tris,

intern/cycles/blender/blender_sync.h

@@ -78,8 +78,6 @@ public:
 	void sync_view(BL::SpaceView3D& b_v3d,
 	               BL::RegionView3D& b_rv3d,
 	               int width, int height);
-	inline int get_layer_samples() { return view_layer.samples; }
-	inline int get_layer_bound_samples() { return view_layer.bound_samples; }
 
 	/* get parameters */
 	static SceneParams get_scene_params(BL::Scene& b_scene,
@@ -126,6 +124,7 @@ private:
 	                 bool motion,
 	                 int motion_step = 0);
 	Object *sync_object(BL::Depsgraph& b_depsgraph,
+	                    BL::ViewLayer& b_view_layer,
 	                    BL::DepsgraphObjectInstance& b_instance,
 	                    float motion_time,
 	                    bool hide_tris,

release/scripts/startup/bl_ui/space_outliner.py

@@ -142,8 +142,12 @@ class OUTLINER_MT_collection_view_layer(Menu):
 
         space = context.space_data
 
-        layout.operator("outliner.collection_exclude_set", text="Exclude")
+        layout.operator("outliner.collection_exclude_set")
-        layout.operator("outliner.collection_include_set", text="Include")
+        layout.operator("outliner.collection_exclude_clear")
+
+        if context.engine == 'CYCLES':
+            layout.operator("outliner.collection_holdout_set")
+            layout.operator("outliner.collection_holdout_clear")
 
 
 class OUTLINER_MT_collection(Menu):

source/blender/blenkernel/intern/layer.c

@@ -715,6 +715,11 @@ static int layer_collection_sync(
 				lc->runtime_flag |= LAYER_COLLECTION_HAS_VISIBLE_OBJECTS;
 			}
 
+			/* Holdout */
+			if (lc->flag & LAYER_COLLECTION_HOLDOUT) {
+				base->flag |= BASE_HOLDOUT;
+			}
+
 			lc->runtime_flag |= LAYER_COLLECTION_HAS_OBJECTS;
 		}
 
@@ -750,7 +755,12 @@ void BKE_layer_collection_sync(const Scene *scene, ViewLayer *view_layer)
 
 	/* Clear visible and selectable flags to be reset. */
 	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
-		base->flag &= ~(BASE_VISIBLE | BASE_ENABLED | BASE_SELECTABLE | BASE_ENABLED_VIEWPORT | BASE_ENABLED_RENDER);
+		base->flag &= ~(BASE_VISIBLE |
+		                BASE_ENABLED |
+		                BASE_SELECTABLE |
+		                BASE_ENABLED_VIEWPORT |
+		                BASE_ENABLED_RENDER |
+		                BASE_HOLDOUT);
 	}
 
 	view_layer->runtime_flag = 0;

source/blender/blenloader/intern/versioning_280.c

@@ -492,6 +492,7 @@ static void do_version_layers_to_collections(Main *bmain, Scene *scene)
 				{
 					if (srl->lay_zmask & (1 << layer)) {
 						have_override = true;
+						lc->flag |= LAYER_COLLECTION_HOLDOUT;
 
 						BKE_override_layer_collection_boolean_add(
 						        lc,

source/blender/editors/space_outliner/outliner_collections.c

@@ -574,7 +574,7 @@ static TreeTraversalAction layer_collection_find_data_to_edit(TreeElement *te, v
 	return TRAVERSE_CONTINUE;
 }
 
-static bool collections_view_layer_poll(bContext *C, bool include)
+static bool collections_view_layer_poll(bContext *C, bool clear, int flag)
 {
 	/* Poll function so the right click menu show current state of selected collections. */
 	SpaceOops *soops = CTX_wm_space_outliner(C);
@@ -593,10 +593,10 @@ static bool collections_view_layer_poll(bContext *C, bool include)
 	GSET_ITER(collections_to_edit_iter, data.collections_to_edit) {
 		LayerCollection *lc = BLI_gsetIterator_getKey(&collections_to_edit_iter);
 
-		if (include && (lc->flag & LAYER_COLLECTION_EXCLUDE)) {
+		if (clear && (lc->flag & flag)) {
 			result = true;
 		}
-		else if (!include && !(lc->flag & LAYER_COLLECTION_EXCLUDE)) {
+		else if (!clear && !(lc->flag & flag)) {
 			result = true;
 		}
 	}
@@ -605,27 +605,37 @@ static bool collections_view_layer_poll(bContext *C, bool include)
 	return result;
 }
 
-static bool collections_exclude_poll(bContext *C)
+static bool collections_exclude_set_poll(bContext *C)
 {
-	return collections_view_layer_poll(C, false);
+	return collections_view_layer_poll(C, false, LAYER_COLLECTION_EXCLUDE);
 }
 
-static bool collections_include_poll(bContext *C)
+static bool collections_exclude_clear_poll(bContext *C)
 {
-	return collections_view_layer_poll(C, true);
+	return collections_view_layer_poll(C, true, LAYER_COLLECTION_EXCLUDE);
 }
 
-static void layer_collection_exclude_recursive_set(LayerCollection *lc)
+static bool collections_holdout_set_poll(bContext *C)
+{
+	return collections_view_layer_poll(C, false, LAYER_COLLECTION_HOLDOUT);
+}
+
+static bool collections_holdout_clear_poll(bContext *C)
+{
+	return collections_view_layer_poll(C, true, LAYER_COLLECTION_HOLDOUT);
+}
+
+static void layer_collection_flag_recursive_set(LayerCollection *lc, int flag)
 {
 	for (LayerCollection *nlc = lc->layer_collections.first; nlc; nlc = nlc->next) {
-		if (lc->flag & LAYER_COLLECTION_EXCLUDE) {
+		if (lc->flag & flag) {
-			nlc->flag |= LAYER_COLLECTION_EXCLUDE;
+			nlc->flag |= flag;
 		}
 		else {
-			nlc->flag &= ~LAYER_COLLECTION_EXCLUDE;
+			nlc->flag &= ~flag;
 		}
 
-		layer_collection_exclude_recursive_set(nlc);
+		layer_collection_flag_recursive_set(nlc, flag);
 	}
 }
 
@@ -636,7 +646,8 @@ static int collection_view_layer_exec(bContext *C, wmOperator *op)
 	ViewLayer *view_layer = CTX_data_view_layer(C);
 	SpaceOops *soops = CTX_wm_space_outliner(C);
 	struct CollectionEditData data = {.scene = scene, .soops = soops};
-	bool include = STREQ(op->idname, "OUTLINER_OT_collection_include_set");
+	bool clear = strstr(op->idname, "clear") != NULL;
+	int flag = strstr(op->idname, "holdout") ? LAYER_COLLECTION_HOLDOUT : LAYER_COLLECTION_EXCLUDE;
 
 	data.collections_to_edit = BLI_gset_ptr_new(__func__);
 
@@ -647,14 +658,14 @@ static int collection_view_layer_exec(bContext *C, wmOperator *op)
 		LayerCollection *lc = BLI_gsetIterator_getKey(&collections_to_edit_iter);
 
 		if (!(lc->collection->flag & COLLECTION_IS_MASTER)) {
-			if (include) {
+			if (clear) {
-				lc->flag &= ~LAYER_COLLECTION_EXCLUDE;
+				lc->flag &= ~flag;
 			}
 			else {
-				lc->flag |= LAYER_COLLECTION_EXCLUDE;
+				lc->flag |= flag;
 			}
 
-			layer_collection_exclude_recursive_set(lc);
+			layer_collection_flag_recursive_set(lc, flag);
 		}
 	}
 
@@ -671,28 +682,58 @@ static int collection_view_layer_exec(bContext *C, wmOperator *op)
 void OUTLINER_OT_collection_exclude_set(wmOperatorType *ot)
 {
 	/* identifiers */
-	ot->name = "Exclude from View Layer";
+	ot->name = "Set Exclude";
 	ot->idname = "OUTLINER_OT_collection_exclude_set";
 	ot->description = "Exclude collection from the active view layer";
 
 	/* api callbacks */
 	ot->exec = collection_view_layer_exec;
-	ot->poll = collections_exclude_poll;
+	ot->poll = collections_exclude_set_poll;
 
 	/* flags */
 	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
-void OUTLINER_OT_collection_include_set(wmOperatorType *ot)
+void OUTLINER_OT_collection_exclude_clear(wmOperatorType *ot)
 {
 	/* identifiers */
-	ot->name = "Include in View Layer";
+	ot->name = "Clear Exclude";
-	ot->idname = "OUTLINER_OT_collection_include_set";
+	ot->idname = "OUTLINER_OT_collection_exclude_clear";
 	ot->description = "Include collection in the active view layer";
 
 	/* api callbacks */
 	ot->exec = collection_view_layer_exec;
-	ot->poll = collections_include_poll;
+	ot->poll = collections_exclude_clear_poll;
+
+	/* flags */
+	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+}
+
+void OUTLINER_OT_collection_holdout_set(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Set Holdout";
+	ot->idname = "OUTLINER_OT_collection_holdout_set";
+	ot->description = "Mask collection in the active view layer";
+
+	/* api callbacks */
+	ot->exec = collection_view_layer_exec;
+	ot->poll = collections_holdout_set_poll;
+
+	/* flags */
+	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+}
+
+void OUTLINER_OT_collection_holdout_clear(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Clear Holdout";
+	ot->idname = "OUTLINER_OT_collection_holdout_clear";
+	ot->description = "Clear masking of collection in the active view layer";
+
+	/* api callbacks */
+	ot->exec = collection_view_layer_exec;
+	ot->poll = collections_holdout_clear_poll;
 
 	/* flags */
 	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;

source/blender/editors/space_outliner/outliner_intern.h

@@ -358,7 +358,9 @@ void OUTLINER_OT_collection_objects_deselect(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_link(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_instance(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_exclude_set(struct wmOperatorType *ot);
-void OUTLINER_OT_collection_include_set(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_exclude_clear(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_holdout_set(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_holdout_clear(struct wmOperatorType *ot);
 
 /* outliner_utils.c ---------------------------------------------- */
 

source/blender/editors/space_outliner/outliner_ops.c

@@ -457,7 +457,9 @@ void outliner_operatortypes(void)
 	WM_operatortype_append(OUTLINER_OT_collection_link);
 	WM_operatortype_append(OUTLINER_OT_collection_instance);
 	WM_operatortype_append(OUTLINER_OT_collection_exclude_set);
-	WM_operatortype_append(OUTLINER_OT_collection_include_set);
+	WM_operatortype_append(OUTLINER_OT_collection_exclude_clear);
+	WM_operatortype_append(OUTLINER_OT_collection_holdout_set);
+	WM_operatortype_append(OUTLINER_OT_collection_holdout_clear);
 }
 
 static wmKeyMap *outliner_item_drag_drop_modal_keymap(wmKeyConfig *keyconf)
@@ -571,7 +573,7 @@ void outliner_keymap(wmKeyConfig *keyconf)
 	WM_keymap_add_item(keymap, "OBJECT_OT_link_to_collection", MKEY, KM_PRESS, KM_SHIFT, 0);
 
 	WM_keymap_add_item(keymap, "OUTLINER_OT_collection_exclude_set", EKEY, KM_PRESS, 0, 0);
-	WM_keymap_add_item(keymap, "OUTLINER_OT_collection_include_set", EKEY, KM_PRESS, KM_ALT, 0);
+	WM_keymap_add_item(keymap, "OUTLINER_OT_collection_exclude_clear", EKEY, KM_PRESS, KM_ALT, 0);
 
 	kmi = WM_keymap_add_item(keymap, "OBJECT_OT_hide_view_clear", HKEY, KM_PRESS, KM_ALT, 0);
 	RNA_boolean_set(kmi->ptr, "select", false);

source/blender/makesdna/DNA_layer_types.h

@@ -104,6 +104,7 @@ enum {
 	BASE_ENABLED_VIEWPORT = (1 << 6), /* Object is enabled in viewport. */
 	BASE_ENABLED_RENDER   = (1 << 7), /* Object is enabled in final render */
 	BASE_ENABLED          = (1 << 9), /* Object is enabled. */
+	BASE_HOLDOUT          = (1 << 10), /* Object masked out from render */
 };
 
 /* LayerCollection->flag */
@@ -113,6 +114,7 @@ enum {
 	/* LAYER_COLLECTION_DEPRECATED2 = (1 << 2), */
 	/* LAYER_COLLECTION_DEPRECATED3 = (1 << 3), */
 	LAYER_COLLECTION_EXCLUDE = (1 << 4),
+	LAYER_COLLECTION_HOLDOUT = (1 << 5),
 };
 
 /* Layer Collection->runtime_flag */

source/blender/makesrna/intern/rna_layer.c

@@ -230,6 +230,12 @@ static void rna_def_layer_collection(BlenderRNA *brna)
 	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
 	RNA_def_property_ui_text(prop, "Exclude", "Exclude collection from view layer");
 	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
+
+	prop = RNA_def_property(srna, "holdout", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", LAYER_COLLECTION_HOLDOUT);
+	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+	RNA_def_property_ui_text(prop, "Holdout", "Mask out collection from view layer");
+	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
 }
 
 static void rna_def_layer_collections(BlenderRNA *brna, PropertyRNA *cprop)

source/blender/makesrna/intern/rna_object_api.c

@@ -148,6 +148,18 @@ static bool rna_Object_visible_get(Object *ob, bContext *C, ReportList *reports)
 	return ((base->flag & BASE_VISIBLE) != 0) ? 1 : 0;
 }
 
+static bool rna_Object_holdout_get(Object *ob, ReportList *reports, ViewLayer *view_layer)
+{
+	Base *base = BKE_view_layer_base_find(view_layer, ob);
+
+	if (!base) {
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		return -1;
+	}
+
+	return ((base->flag & BASE_HOLDOUT) != 0) ? 1 : 0;
+}
+
 /* Convert a given matrix from a space to another (using the object and/or a bone as reference). */
 static void rna_Object_mat_convert_space(Object *ob, ReportList *reports, bPoseChannel *pchan,
                                         float *mat, float *mat_ret, int from, int to)
@@ -470,31 +482,39 @@ void RNA_api_object(StructRNA *srna)
 #endif
 
 	static EnumPropertyItem object_select_items[] = {
-	    {0, "SELECT", 0, "Select", "Select object from the active render layer"},
+	    {0, "SELECT", 0, "Select", "Select object from the active view layer"},
-	    {1, "DESELECT", 0, "Deselect", "Deselect object from the active render layer"},
+	    {1, "DESELECT", 0, "Deselect", "Deselect object from the active view layer"},
-	    {2, "TOGGLE", 0, "Toggle", "Toggle object selection from the active render layer"},
+	    {2, "TOGGLE", 0, "Toggle", "Toggle object selection from the active view layer"},
 	    {0, NULL, 0, NULL, NULL}
 	};
 
 	/* Special wrapper to access the base selection value */
 	func = RNA_def_function(srna, "select_set", "rna_Object_select_set");
-	RNA_def_function_ui_description(func, "Select the object (for the active render layer)");
+	RNA_def_function_ui_description(func, "Select the object (for the active view layer)");
 	RNA_def_function_flag(func, FUNC_USE_CONTEXT | FUNC_USE_REPORTS);
 	parm = RNA_def_enum(func, "action", object_select_items, 0, "Action", "Select mode");
 	RNA_def_parameter_flags(parm, 0, PARM_REQUIRED);
 
 	func = RNA_def_function(srna, "select_get", "rna_Object_select_get");
-	RNA_def_function_ui_description(func, "Get the object selection for the active render layer");
+	RNA_def_function_ui_description(func, "Get the object selection for the active view layer");
 	RNA_def_function_flag(func, FUNC_USE_CONTEXT | FUNC_USE_REPORTS);
 	parm = RNA_def_boolean(func, "result", 0, "", "Object selected");
 	RNA_def_function_return(func, parm);
 
 	func = RNA_def_function(srna, "visible_get", "rna_Object_visible_get");
-	RNA_def_function_ui_description(func, "Get the object visibility for the active render layer");
+	RNA_def_function_ui_description(func, "Get the object visibility for the active view layer");
 	RNA_def_function_flag(func, FUNC_USE_CONTEXT | FUNC_USE_REPORTS);
 	parm = RNA_def_boolean(func, "result", 0, "", "Object visible");
 	RNA_def_function_return(func, parm);
 
+	func = RNA_def_function(srna, "holdout_get", "rna_Object_holdout_get");
+	RNA_def_function_ui_description(func, "Test if object is masked in the view layer");
+	RNA_def_function_flag(func, FUNC_USE_REPORTS);
+	parm = RNA_def_pointer(func, "view_layer", "ViewLayer", "", "");
+	RNA_def_parameter_flags(parm, 0, PARM_REQUIRED);
+	parm = RNA_def_boolean(func, "result", 0, "", "Object holdout");
+	RNA_def_function_return(func, parm);
+
 	/* Matrix space conversion */
 	func = RNA_def_function(srna, "convert_space", "rna_Object_mat_convert_space");
 	RNA_def_function_ui_description(func, "Convert (transform) the given matrix from one space to another");