GPU: Add imageLoadFast and imageStoreFast variants

Add fast image writing and reading variants for additional use
cases. These variants do not perform range checking on values
and should only be used in cases where the written texel is
guaranteed to be in range. This eliminates additional
branching and simplifies shader logic.

Single-channel write variants have also been added to
match single channel write variants exposed by the Metal
backend, reducing temporary register requirements.

Authored by Apple: Michael Parkin-White

@@ -15,0 +15,4 @@
+#define imageLoadFast imageLoad
+/* Fast store variant for single-channel writes. Special case which avoids unnecessary vector
+ * pack/unpback. Passthrough in GLSL. */
+#define imageStoreFast_1chFloat(tex, px, val) imageStore(tex, px, vec4(val))

@@ -33,3 +33,3 @@
   vec4 out_color = weighted_sum_array(colors, weights);
 
-  imageStore(out_color_img, ivec2(gl_GlobalInvocationID.xy), out_color);
+  imageStoreFast(out_color_img, ivec2(gl_GlobalInvocationID.xy), out_color);

@@ -168,2 +168,2 @@
-  imageStore(out_color_img, out_texel, median.color);
-  imageStore(out_weight_img, out_texel, vec4(median.weight));
+  imageStoreFast(out_color_img, out_texel, median.color);
+  imageStoreFast(out_weight_img, out_texel, vec4(median.weight));

@@ -101,1 +99,3 @@
-  imageStore(out_occlusion_img, out_texel, out_occlusion.xyxy);
+  imageStoreFast(out_color_img, out_texel, out_color);
+  imageStoreFast(out_weight_img, out_texel, vec4(out_weight));
+  imageStoreFast(out_occlusion_img, out_texel, out_occlusion.xyxy);

@@ -70,3 +70,2 @@
   ivec2 out_texel = ivec2(gl_GlobalInvocationID.xy);
-  imageStore(out_color_img, out_texel, out_color);
-  imageStore(out_weight_img, out_texel, vec4(out_weight));
+  imageStoreFast(out_color_img, out_texel, out_color);

@@ -246,2 +246,2 @@
-        imageStore(out_color_lod3_img, texel, color_cache[LOCAL_INDEX]);
-        imageStore(out_coc_lod3_img, texel, vec4(coc_cache[LOCAL_INDEX]));
+        imageStoreFast(out_color_lod3_img, texel, color_cache[LOCAL_INDEX]);
+        imageStoreFast(out_coc_lod3_img, texel, vec4(coc_cache[LOCAL_INDEX]));

@@ -44,3 +44,3 @@
   ivec2 out_texel = ivec2(gl_GlobalInvocationID.xy);
   vec4 out_color = weighted_sum_array(colors, weights);
-  imageStore(out_color_img, out_texel, out_color);
+  imageStoreFast(out_color_img, out_texel, out_color);

@@ -368,3 +368,3 @@
 
   /* Save history for next iteration. Still in YCoCg space with CoC in alpha. */
-  imageStore(out_history_img, src_texel, result.color);
+  imageStoreFast(out_history_img, src_texel, result.color);

@@ -72,3 +72,3 @@
     cryptomatte_sort_samples(samples);
     /* Repeat texture coordinates as the weight can be optimized to a small portion of the film. */
-    float weight = imageLoad(
+    float weight = imageLoadFast(

@@ -672,3 +672,3 @@
           uniform_buf.film.display_id == uniform_buf.film.normal_id)
       {
-        out_color = imageLoad(color_accum_img, ivec3(texel_film, uniform_buf.film.display_id));
+        out_color = imageLoadFast(color_accum_img, ivec3(texel_film, uniform_buf.film.display_id));

@@ -91,2 +91,2 @@
-  float variance = imageLoad(in_variance_img, texel_fullres).r;
-  vec3 in_radiance = imageLoad(in_radiance_img, texel_fullres).rgb;
+  float variance = imageLoadFast(in_variance_img, texel_fullres).r;
+  vec3 in_radiance = imageLoadFast(in_radiance_img, texel_fullres).rgb;

@@ -123,3 +123,3 @@
     ivec3 sample_tile = ivec3(sample_texel / RAYTRACE_GROUP_SIZE, closure_index);
     /* Make sure the sample has been processed and do not contain garbage data. */
-    if (imageLoad(tile_mask_img, sample_tile).r == 0u) {
+    if (imageLoadFast(tile_mask_img, sample_tile).r == 0u) {

@@ -136,3 +136,3 @@
     }
 
-    vec3 radiance = imageLoad(in_radiance_img, sample_texel).rgb;
+    vec3 radiance = imageLoadFast(in_radiance_img, sample_texel).rgb;

@@ -164,3 +164,3 @@
   out_radiance = from_accumulation_space(out_radiance);
 
-  imageStore(out_radiance_img, texel_fullres, vec4(out_radiance, 0.0));
+  imageStoreFast(out_radiance_img, texel_fullres, vec4(out_radiance, 0.0));

@@ -76,3 +76,3 @@
 #endif
   if (do_skip_denoise) {
-    imageStore(out_radiance_img, texel_fullres, imageLoad(ray_radiance_img, texel));
+    imageStoreFast(out_radiance_img, texel_fullres, imageLoadFast(ray_radiance_img, texel));

@@ -99,3 +99,3 @@
       ivec3 sample_tile = ivec3(tile_coord_neighbor, closure_index);
 
-      uint tile_mask = imageLoad(tile_mask_img, sample_tile).r;
+      uint tile_mask = imageLoadFast(tile_mask_img, sample_tile).r;

@@ -193,2 +192,2 @@
-  imageStore(out_variance_img, texel_fullres, vec4(hit_variance));
-  imageStore(out_hit_depth_img, texel_fullres, vec4(hit_depth));
+  imageStoreFast(out_radiance_img, texel_fullres, vec4(radiance_accum, 0.0));
+  imageStoreFast(out_variance_img, texel_fullres, vec4(hit_variance));

@@ -177,3 +177,2 @@
 
-  float in_variance = imageLoad(in_variance_img, texel_fullres).r;
-  vec3 in_radiance = imageLoad(in_radiance_img, texel_fullres).rgb;
+  float in_variance = imageLoadFast(in_variance_img, texel_fullres).r;

@@ -27,3 +27,3 @@
   bool valid_pixel = closure_index < gbuf.closure_count;
   if (!valid_pixel) {
-    imageStore(out_ray_data_img, texel, vec4(0.0));
+    imageStoreFast(out_ray_data_img, texel, vec4(0.0));

@@ -43,3 +43,3 @@
    * Strangely it does not correspond to the IEEE spec. */
   float inv_pdf = (samp.pdf == 0.0) ? 0.0 : max(6e-8, 1.0 / samp.pdf);
-  imageStore(out_ray_data_img, texel, vec4(samp.direction, inv_pdf));
+  imageStoreFast(out_ray_data_img, texel, vec4(samp.direction, inv_pdf));

@@ -26,3 +26,2 @@
 
-  vec4 ray_data = imageLoad(ray_data_img, texel);
-  float ray_pdf_inv = ray_data.w;
+  vec4 ray_data_im = imageLoadFast(ray_data_img, texel);

@@ -22,13 +22,13 @@ void main()
   uvec2 tile_coord = unpackUvec2x16(tiles_coord_buf[gl_WorkGroupID.x]);
   ivec2 texel = ivec2(gl_LocalInvocationID.xy + tile_coord * tile_size);
 
-  vec4 ray_data = imageLoad(ray_data_img, texel);

@@ -24,3 +24,2 @@
 
-  vec4 ray_data = imageLoad(ray_data_img, texel);
-  float ray_pdf_inv = ray_data.w;
+  vec4 ray_data_im = imageLoadFast(ray_data_img, texel);

@@ -19,8 +19,8 @@ void main()
   uvec2 tile_coord = unpackUvec2x16(tiles_coord_buf[gl_WorkGroupID.x]);
   ivec2 texel = ivec2(gl_LocalInvocationID.xy + tile_coord * tile_size);
 
-  vec4 ray_data = imageLoad(ray_data_img, texel);

@@ -21,3 +21,2 @@
 
-  vec4 ray_data = imageLoad(ray_data_img, texel);
-  float ray_pdf_inv = ray_data.w;
+  vec4 ray_data_im = imageLoadFast(ray_data_img, texel);

@@ -8,3 +8,3 @@
   if (id >= 0) {
     ivec2 texel = ivec2(gl_FragCoord.xy);
-    imageStore(rp_color_img, ivec3(texel, id), color);
+    imageStoreFast(rp_color_img, ivec3(texel, id), color);

@@ -135,2 +135,4 @@
           view_infos_buf[view_index].winmat = winmat;
           view_infos_buf[view_index].wininv = inverse(winmat);
+          /* NOTE: We may not end up using this due to potential imprecision. */
+          view_infos_buf[view_index].persmat_sh = winmat * tilemap_data.viewmat;

@@ -182,3 +184,3 @@
   /* Store the highest LOD valid page for rendering. */
   uint tile_packed = (valid_tile_index != -1) ? tiles_buf[valid_tile_index] : SHADOW_NO_DATA;
-  imageStore(tilemaps_img, atlas_texel, uvec4(tile_packed));
+  imageStoreFast(tilemaps_img, atlas_texel, uvec4(tile_packed));

@@ -34,3 +34,2 @@
 
-    imageStore(radiance_img, texel, vec4(radiance, 0.0));
-    imageStore(object_id_img, texel, uvec4(gbuf.object_id));
+    imageStoreFast(radiance_img, texel, vec4(radiance, 0.0));

@@ -50,3 +50,3 @@
   else {
     /* No need to write radiance_img since the radiance won't be used at all. */
-    imageStore(object_id_img, texel, uvec4(0));
+    imageStoreFast(object_id_img, texel, uvec4(0));