Fix T80862: Small stroke opacity modulation is best done in the fragment shader

To avoid anti-aliasing artifacts on GPencil strokes that have a size smaller than 1 the thickness
is clamped and the opacity reduced. This was done in vertex shader but this had the side effect
of causing strokes that go from large to small to fade across their lengths.

**Solution**

The opacity modulation has now been moved to the fragment shader as advised by Clément Foucault.

The strokeThickness that was passed to the shader was clamped to prevent it from being too small so an additional
unclamped thickness has been passed to the fragment to calculate the opacity modulation. Alternatively I could have chosen
strokeThickness and clampedThickness but I decided against renaming the variables in the current code.


Reviewed By: fclem

Differential Revision: https://developer.blender.org/D10438

source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl

@@ -156,6 +156,7 @@ IN_OUT ShaderStageInterface
   vec3 finalPos;
   vec2 finalUvs;
   noperspective float strokeThickness;
+  noperspective float unclampedThickness;
   noperspective float strokeHardeness;
   flat vec2 strokeAspect;
   flat vec2 strokePt1;
@@ -324,7 +325,7 @@ vec2 safe_normalize_len(vec2 v, out float len)
   }
 }
 
-float stroke_thickness_modulate(float thickness, out float opacity)
+float stroke_thickness_modulate(float thickness)
 {
   /* Modify stroke thickness by object and layer factors.-*/
   thickness *= thicknessScale;
@@ -340,9 +341,14 @@ float stroke_thickness_modulate(float thickness, out float opacity)
     /* World space point size. */
     thickness *= thicknessWorldScale * ProjectionMatrix[1][1] * sizeViewport.y;
   }
-  /* To avoid aliasing artifact, we clamp the line thickness and reduce its opacity. */
+  return thickness;
+}
+
+float clamp_small_stroke_thickness(float thickness)
+{
+  /* To avoid aliasing artifacts, we clamp the line thickness and
+   * the reduce its opacity in the fragment shader.*/
   float min_thickness = gl_Position.w * 1.3;
-  opacity = smoothstep(0.0, gl_Position.w * 1.0, thickness);
   thickness = max(min_thickness, thickness);
 
   return thickness;
@@ -426,9 +432,9 @@ void stroke_vertex()
   vec2 line = safe_normalize_len(ss2 - ss1, line_len);
   vec2 line_adj = safe_normalize((use_curr) ? (ss1 - ss_adj) : (ss_adj - ss2));
 
-  float small_line_opacity;
   float thickness = abs((use_curr) ? thickness1 : thickness2);
-  thickness = stroke_thickness_modulate(thickness, small_line_opacity);
+  thickness = stroke_thickness_modulate(thickness);
+  float clampedThickness = clamp_small_stroke_thickness(thickness);
 
   finalUvs = vec2(x, y) * 0.5 + 0.5;
   strokeHardeness = decode_hardness(use_curr ? hardness1 : hardness2);
@@ -479,11 +485,12 @@ void stroke_vertex()
     /* Invert for vertex shader. */
     strokeAspect = 1.0 / strokeAspect;
 
-    gl_Position.xy += (x * x_axis + y * y_axis) * sizeViewportInv.xy * thickness;
+    gl_Position.xy += (x * x_axis + y * y_axis) * sizeViewportInv.xy * clampedThickness;
 
     strokePt1 = ss1;
     strokePt2 = ss1 + x_axis * 0.5;
-    strokeThickness = (is_squares) ? 1e18 : (thickness / gl_Position.w);
+    strokeThickness = (is_squares) ? 1e18 : (clampedThickness / gl_Position.w);
+    unclampedThickness = (is_squares) ? 1e18 : (thickness / gl_Position.w);
   }
   else {
     bool is_stroke_start = (ma.x == -1 && x == -1);
@@ -501,7 +508,8 @@ void stroke_vertex()
 
     strokePt1.xy = ss1;
     strokePt2.xy = ss2;
-    strokeThickness = thickness / gl_Position.w;
+    strokeThickness = clampedThickness / gl_Position.w;
+    unclampedThickness = thickness / gl_Position.w;
     strokeAspect = vec2(1.0);
 
     vec2 screen_ofs = miter * y;
@@ -512,7 +520,7 @@ void stroke_vertex()
       screen_ofs += line * x;
     }
 
-    gl_Position.xy += screen_ofs * sizeViewportInv.xy * thickness;
+    gl_Position.xy += screen_ofs * sizeViewportInv.xy * clampedThickness;
 
     finalUvs.x = (use_curr) ? uv1.z : uv2.z;
 #  ifdef GP_MATERIAL_BUFFER_LEN
@@ -531,7 +539,7 @@ void stroke_vertex()
     vert_col = vec4(0.0);
   }
 
-  color_output(stroke_col, vert_col, vert_strength * small_line_opacity, mix_tex);
+  color_output(stroke_col, vert_col, vert_strength, mix_tex);
 
   matFlag = GP_FLAG(m) & ~GP_FILL_FLAGS;
 #  endif

source/blender/draw/engines/gpencil/shaders/gpencil_frag.glsl

@@ -89,6 +89,9 @@ void main()
   fragColor *= stroke_round_cap_mask(
       strokePt1, strokePt2, strokeAspect, strokeThickness, strokeHardeness);
 
+  /* To avoid aliasing artifacts, we reduce the opacity of small strokes. */
+  fragColor *= smoothstep(0.0, 1.0, unclampedThickness);
+
   /* Holdout materials. */
   if (GP_FLAG_TEST(matFlag, GP_STROKE_HOLDOUT | GP_FILL_HOLDOUT)) {
     revealColor = fragColor.aaaa;