Cycles: add per layer collection indirectly on setting.

In the outliner, right click > view layer > set indirect only. This is
like clearing camera ray visibility on objects in the collection, and is
temporary until we have more general dynamic overrides.

intern/cycles/blender/blender_object.cpp

@@ -361,12 +361,11 @@ Object *BlenderSync::sync_object(BL::Depsgraph& b_depsgraph,
 	}
 #endif
 
-	/* TODO: hide objects not on render layer from camera rays. */
-#if 0
-	if(!(layer_flag & view_layer.layer)) {
+	/* Clear camera visibility for indirect only objects. */
+	bool use_indirect_only = b_ob.indirect_only_get(b_view_layer);
+	if(use_indirect_only) {
 		visibility &= ~PATH_RAY_CAMERA;
 	}
-#endif
 
 	/* Don't export completely invisible objects. */
 	if(visibility == 0) {

release/scripts/startup/bl_ui/space_outliner.py

@@ -146,6 +146,9 @@ class OUTLINER_MT_collection_view_layer(Menu):
         layout.operator("outliner.collection_exclude_clear")
 
         if context.engine == 'CYCLES':
+            layout.operator("outliner.collection_indirect_only_set")
+            layout.operator("outliner.collection_indirect_only_clear")
+
             layout.operator("outliner.collection_holdout_set")
             layout.operator("outliner.collection_holdout_clear")
 

source/blender/blenkernel/intern/layer.c

@@ -715,10 +715,13 @@ static int layer_collection_sync(
 				lc->runtime_flag |= LAYER_COLLECTION_HAS_VISIBLE_OBJECTS;
 			}
 
-			/* Holdout */
+			/* Holdout and indirect only */
 			if (lc->flag & LAYER_COLLECTION_HOLDOUT) {
 				base->flag |= BASE_HOLDOUT;
 			}
+			if (lc->flag & LAYER_COLLECTION_INDIRECT_ONLY) {
+				base->flag |= BASE_INDIRECT_ONLY;
+			}
 
 			lc->runtime_flag |= LAYER_COLLECTION_HAS_OBJECTS;
 		}
@@ -760,7 +763,8 @@ void BKE_layer_collection_sync(const Scene *scene, ViewLayer *view_layer)
 		                BASE_SELECTABLE |
 		                BASE_ENABLED_VIEWPORT |
 		                BASE_ENABLED_RENDER |
-		                BASE_HOLDOUT);
+		                BASE_HOLDOUT |
+		                BASE_INDIRECT_ONLY);
 	}
 
 	view_layer->runtime_flag = 0;

source/blender/blenloader/intern/versioning_280.c

@@ -487,28 +487,15 @@ static void do_version_layers_to_collections(Main *bmain, Scene *scene)
 						nlc->flag |= LAYER_COLLECTION_EXCLUDE;
 					}
 				}
-				else if ((scene->lay & srl->lay & ~(srl->lay_exclude) & (1 << layer)) ||
-				         (srl->lay_zmask & (scene->lay | srl->lay_exclude) & (1 << layer)))
-				{
+				else {
 					if (srl->lay_zmask & (1 << layer)) {
 						have_override = true;
 						lc->flag |= LAYER_COLLECTION_HOLDOUT;
-
-						BKE_override_layer_collection_boolean_add(
-						        lc,
-						        ID_OB,
-						        "cycles.is_holdout",
-						        true);
 					}
 
 					if ((srl->lay & (1 << layer)) == 0) {
 						have_override = true;
-
-						BKE_override_layer_collection_boolean_add(
-						        lc,
-						        ID_OB,
-						        "cycles_visibility.camera",
-						        false);
+						lc->flag |= LAYER_COLLECTION_INDIRECT_ONLY;
 					}
 				}
 			}

source/blender/editors/space_outliner/outliner_collections.c

@@ -625,6 +625,16 @@ static bool collections_holdout_clear_poll(bContext *C)
 	return collections_view_layer_poll(C, true, LAYER_COLLECTION_HOLDOUT);
 }
 
+static bool collections_indirect_only_set_poll(bContext *C)
+{
+	return collections_view_layer_poll(C, false, LAYER_COLLECTION_INDIRECT_ONLY);
+}
+
+static bool collections_indirect_only_clear_poll(bContext *C)
+{
+	return collections_view_layer_poll(C, true, LAYER_COLLECTION_INDIRECT_ONLY);
+}
+
 static void layer_collection_flag_recursive_set(LayerCollection *lc, int flag)
 {
 	for (LayerCollection *nlc = lc->layer_collections.first; nlc; nlc = nlc->next) {
@@ -647,7 +657,9 @@ static int collection_view_layer_exec(bContext *C, wmOperator *op)
 	SpaceOops *soops = CTX_wm_space_outliner(C);
 	struct CollectionEditData data = {.scene = scene, .soops = soops};
 	bool clear = strstr(op->idname, "clear") != NULL;
-	int flag = strstr(op->idname, "holdout") ? LAYER_COLLECTION_HOLDOUT : LAYER_COLLECTION_EXCLUDE;
+	int flag = strstr(op->idname, "holdout") ?       LAYER_COLLECTION_HOLDOUT :
+	           strstr(op->idname, "indirect_only") ? LAYER_COLLECTION_INDIRECT_ONLY :
+	                                                 LAYER_COLLECTION_EXCLUDE;
 
 	data.collections_to_edit = BLI_gset_ptr_new(__func__);
 
@@ -739,6 +751,36 @@ void OUTLINER_OT_collection_holdout_clear(wmOperatorType *ot)
 	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
+void OUTLINER_OT_collection_indirect_only_set(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Set Indirect Only";
+	ot->idname = "OUTLINER_OT_collection_indirect_only_set";
+	ot->description = "Set collection to only contribute indirectly (through shadows and reflections) in the view layer";
+
+	/* api callbacks */
+	ot->exec = collection_view_layer_exec;
+	ot->poll = collections_indirect_only_set_poll;
+
+	/* flags */
+	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+}
+
+void OUTLINER_OT_collection_indirect_only_clear(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Clear Indirect Only";
+	ot->idname = "OUTLINER_OT_collection_indirect_only_clear";
+	ot->description = "Clear collection contributing only indirectly in the view layer";
+
+	/* api callbacks */
+	ot->exec = collection_view_layer_exec;
+	ot->poll = collections_indirect_only_clear_poll;
+
+	/* flags */
+	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+}
+
 /**
  * Populates the \param objects ListBase with all the outliner selected objects
  * We store it as (Object *)LinkData->data

source/blender/editors/space_outliner/outliner_intern.h

@@ -361,6 +361,8 @@ void OUTLINER_OT_collection_exclude_set(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_exclude_clear(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_holdout_set(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_holdout_clear(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_indirect_only_set(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_indirect_only_clear(struct wmOperatorType *ot);
 
 /* outliner_utils.c ---------------------------------------------- */
 

source/blender/editors/space_outliner/outliner_ops.c

@@ -460,6 +460,8 @@ void outliner_operatortypes(void)
 	WM_operatortype_append(OUTLINER_OT_collection_exclude_clear);
 	WM_operatortype_append(OUTLINER_OT_collection_holdout_set);
 	WM_operatortype_append(OUTLINER_OT_collection_holdout_clear);
+	WM_operatortype_append(OUTLINER_OT_collection_indirect_only_set);
+	WM_operatortype_append(OUTLINER_OT_collection_indirect_only_clear);
 }
 
 static wmKeyMap *outliner_item_drag_drop_modal_keymap(wmKeyConfig *keyconf)

source/blender/makesdna/DNA_layer_types.h

@@ -105,6 +105,7 @@ enum {
 	BASE_ENABLED_RENDER   = (1 << 7), /* Object is enabled in final render */
 	BASE_ENABLED          = (1 << 9), /* Object is enabled. */
 	BASE_HOLDOUT          = (1 << 10), /* Object masked out from render */
+	BASE_INDIRECT_ONLY    = (1 << 11), /* Object only contributes indirectly to render */
 };
 
 /* LayerCollection->flag */
@@ -115,6 +116,7 @@ enum {
 	/* LAYER_COLLECTION_DEPRECATED3 = (1 << 3), */
 	LAYER_COLLECTION_EXCLUDE = (1 << 4),
 	LAYER_COLLECTION_HOLDOUT = (1 << 5),
+	LAYER_COLLECTION_INDIRECT_ONLY = (1 << 6),
 };
 
 /* Layer Collection->runtime_flag */

source/blender/makesrna/intern/rna_layer.c

@@ -234,7 +234,13 @@ static void rna_def_layer_collection(BlenderRNA *brna)
 	prop = RNA_def_property(srna, "holdout", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", LAYER_COLLECTION_HOLDOUT);
 	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
-	RNA_def_property_ui_text(prop, "Holdout", "Mask out collection from view layer");
+	RNA_def_property_ui_text(prop, "Holdout", "Mask out objects in collection from view layer");
+	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
+
+	prop = RNA_def_property(srna, "indirect_only", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", LAYER_COLLECTION_INDIRECT_ONLY);
+	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+	RNA_def_property_ui_text(prop, "Indirect Only", "Objects in collection only contribute indirectly (through shadows and reflections) in the view layer");
 	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
 }
 

source/blender/makesrna/intern/rna_object_api.c

@@ -128,7 +128,7 @@ static bool rna_Object_select_get(Object *ob, bContext *C, ReportList *reports)
 	Base *base = BKE_view_layer_base_find(view_layer, ob);
 
 	if (!base) {
-		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
 		return -1;
 	}
 
@@ -141,7 +141,7 @@ static bool rna_Object_visible_get(Object *ob, bContext *C, ReportList *reports)
 	Base *base = BKE_view_layer_base_find(view_layer, ob);
 
 	if (!base) {
-		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
 		return -1;
 	}
 
@@ -153,13 +153,25 @@ static bool rna_Object_holdout_get(Object *ob, ReportList *reports, ViewLayer *v
 	Base *base = BKE_view_layer_base_find(view_layer, ob);
 
 	if (!base) {
-		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
 		return -1;
 	}
 
 	return ((base->flag & BASE_HOLDOUT) != 0) ? 1 : 0;
 }
 
+static bool rna_Object_indirect_only_get(Object *ob, ReportList *reports, ViewLayer *view_layer)
+{
+	Base *base = BKE_view_layer_base_find(view_layer, ob);
+
+	if (!base) {
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
+		return -1;
+	}
+
+	return ((base->flag & BASE_INDIRECT_ONLY) != 0) ? 1 : 0;
+}
+
 /* Convert a given matrix from a space to another (using the object and/or a bone as reference). */
 static void rna_Object_mat_convert_space(Object *ob, ReportList *reports, bPoseChannel *pchan,
                                         float *mat, float *mat_ret, int from, int to)
@@ -515,6 +527,14 @@ void RNA_api_object(StructRNA *srna)
 	parm = RNA_def_boolean(func, "result", 0, "", "Object holdout");
 	RNA_def_function_return(func, parm);
 
+	func = RNA_def_function(srna, "indirect_only_get", "rna_Object_indirect_only_get");
+	RNA_def_function_ui_description(func, "Test if object is set to contribute only indirectly (through shadows and reflections) in the view layer");
+	RNA_def_function_flag(func, FUNC_USE_REPORTS);
+	parm = RNA_def_pointer(func, "view_layer", "ViewLayer", "", "");
+	RNA_def_parameter_flags(parm, 0, PARM_REQUIRED);
+	parm = RNA_def_boolean(func, "result", 0, "", "Object indirect only");
+	RNA_def_function_return(func, parm);
+
 	/* Matrix space conversion */
 	func = RNA_def_function(srna, "convert_space", "rna_Object_mat_convert_space");
 	RNA_def_function_ui_description(func, "Convert (transform) the given matrix from one space to another");