Cycles: Add reflection fix to Bump and Normal Map nodes

While changing the shading normal is a great way to add additional detail to a model, there are some problems with it.
One of them is that at grazing angles and/or strong changes to the normal, the reflected ray can end up pointing into the actual geometry, which results in a black spot.

This patch helps avoid this by automatically reducing the strength of the bump/normal map if the reflected direction would end up too shallow or inside the geometry.

Differential Revision: https://developer.blender.org/D2574

intern/cycles/kernel/kernel_montecarlo.h

@@ -184,6 +184,35 @@ ccl_device float2 regular_polygon_sample(float corners, float rotation, float u,
 	return make_float2(cr*p.x - sr*p.y, sr*p.x + cr*p.y);
 }
 
+ccl_device float3 ensure_valid_reflection(float3 Ng, float3 I, float3 N)
+{
+	float3 R = 2*dot(N, I)*N - I;
+	if(dot(Ng, R) >= 0.05f) {
+		return N;
+	}
+
+	/* Form coordinate system with Ng as the Z axis and N inside the X-Z-plane.
+	 * The X axis is found by normalizing the component of N that's orthogonal to Ng.
+	 * The Y axis isn't actually needed.
+	 */
+	float3 X = normalize(N - dot(N, Ng)*Ng);
+
+	/* Calculate N.z and N.x in the local coordinate system. */
+	float Iz = dot(I, Ng);
+	float Ix2 = sqr(dot(I, X)), Iz2 = sqr(Iz);
+	float Ix2Iz2 = Ix2 + Iz2;
+
+	float a = sqrtf(Ix2*(Ix2Iz2 - sqr(0.05f)));
+	float b = Iz*0.05f + Ix2Iz2;
+	float c = (a + b > 0.0f)? (a + b) : (-a + b);
+
+	float Nz = sqrtf(0.5f * c * (1.0f / Ix2Iz2));
+	float Nx = sqrtf(1.0f - sqr(Nz));
+
+	/* Transform back into global coordinates. */
+	return Nx*X + Nz*Ng;
+}
+
 CCL_NAMESPACE_END
 
 #endif /* __KERNEL_MONTECARLO_CL__ */

intern/cycles/kernel/shaders/node_bump.osl

@@ -64,5 +64,7 @@ surface node_bump(
 	if (use_object_space) {
 		NormalOut = normalize(transform("object", "world", NormalOut));
 	}
+
+	NormalOut = ensure_valid_reflection(Ng, I, NormalOut);
 }
 

intern/cycles/kernel/shaders/node_normal_map.osl

@@ -88,5 +88,7 @@ shader node_normal_map(
 
 	if (Strength != 1.0)
 		Normal = normalize(NormalIn + (Normal - NormalIn) * max(Strength, 0.0));
+
+	Normal = ensure_valid_reflection(Ng, I, Normal);
 }
 

intern/cycles/kernel/shaders/stdosl.h

@@ -282,6 +282,36 @@ point rotate (point p, float angle, point a, point b)
     return transform (M, p-a) + a;
 }
 
+normal ensure_valid_reflection(normal Ng, vector I, normal N)
+{
+    float sqr(float x) { return x*x; }
+
+    vector R = 2*dot(N, I)*N - I;
+    if (dot(Ng, R) >= 0.05) {
+        return N;
+    }
+
+    /* Form coordinate system with Ng as the Z axis and N inside the X-Z-plane.
+     * The X axis is found by normalizing the component of N that's orthogonal to Ng.
+     * The Y axis isn't actually needed.
+     */
+    vector X = normalize(N - dot(N, Ng)*Ng);
+
+    /* Calculate N.z and N.x in the local coordinate system. */
+    float Ix = dot(I, X), Iz = dot(I, Ng);
+    float Ix2 = sqr(dot(I, X)), Iz2 = sqr(dot(I, Ng));
+    float Ix2Iz2 = Ix2 + Iz2;
+
+    float a = sqrt(Ix2*(Ix2Iz2 - sqr(0.05)));
+    float b = Iz*0.05 + Ix2Iz2;
+    float c = (a + b > 0.0)? (a + b) : (-a + b);
+
+    float Nz = sqrt(0.5 * c * (1.0 / Ix2Iz2));
+    float Nx = sqrt(1.0 - sqr(Nz));
+
+    /* Transform back into global coordinates. */
+    return Nx*X + Nz*Ng;
+}
 
 
 // Color functions

intern/cycles/kernel/svm/svm_displace.h

@@ -75,6 +75,8 @@ ccl_device void svm_node_set_bump(KernelGlobals *kg, ShaderData *sd, float *stac
 		object_normal_transform(kg, sd, &normal_out);
 	}
 
+	normal_out = ensure_valid_reflection(sd->Ng, sd->I, normal_out);
+
 	stack_store_float3(stack, node.w, normal_out);
 #endif
 }

intern/cycles/kernel/svm/svm_tex_coord.h

@@ -345,6 +345,8 @@ ccl_device void svm_node_normal_map(KernelGlobals *kg, ShaderData *sd, float *st
 		N = safe_normalize(sd->N + (N - sd->N)*strength);
 	}
 
+	N = ensure_valid_reflection(sd->Ng, sd->I, N);
+
 	if(is_zero(N)) {
 		N = sd->N;
 	}