From 4631b2e9ce86454b84aec1b06b58b9f7afe3aaa9 Mon Sep 17 00:00:00 2001
From: Jeroen Bakker <jeroen@blender.org>
Date: Tue, 11 Apr 2023 10:28:19 +0200
Subject: [PATCH 1/4] GPU: Use Same Type in Comparisons.

Fix: #106432
---
 .../shaders/infos/overlay_edit_mode_info.hh   |  6 +++---
 .../overlay_edit_curve_handle_geom.glsl       | 20 +++++++++----------
 ...verlay_edit_curve_handle_vert_no_geom.glsl | 20 +++++++++----------
 .../overlay_edit_curve_point_vert.glsl        |  6 +++---
 .../overlay_edit_lattice_point_vert.glsl      |  4 ++--
 5 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh b/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh
index cc6ae57a60e..fe03952bbc8 100644
--- a/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh
+++ b/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh
@@ -396,7 +396,7 @@ GPU_SHADER_CREATE_INFO(overlay_edit_curve_handle)
     .geometry_layout(PrimitiveIn::LINES, PrimitiveOut::TRIANGLE_STRIP, 10)
     .geometry_out(overlay_edit_smooth_color_iface)
     .push_constant(Type::BOOL, "showCurveHandles")
-    .push_constant(Type::INT, "curveHandleDisplay")
+    .push_constant(Type::UINT, "curveHandleDisplay")
     .fragment_out(0, Type::VEC4, "fragColor")
     .vertex_source("overlay_edit_curve_handle_vert.glsl")
     .geometry_source("overlay_edit_curve_handle_geom.glsl")
@@ -414,7 +414,7 @@ GPU_SHADER_CREATE_INFO(overlay_edit_curve_handle_no_geom)
     .vertex_in(1, Type::UCHAR, "data")
     .vertex_out(overlay_edit_smooth_color_iface)
     .push_constant(Type::BOOL, "showCurveHandles")
-    .push_constant(Type::INT, "curveHandleDisplay")
+    .push_constant(Type::UINT, "curveHandleDisplay")
     .fragment_out(0, Type::VEC4, "fragColor")
     .vertex_source("overlay_edit_curve_handle_vert_no_geom.glsl")
     .fragment_source("overlay_varying_color.glsl")
@@ -439,7 +439,7 @@ GPU_SHADER_CREATE_INFO(overlay_edit_curve_point)
     .vertex_in(1, Type::UINT, "data")
     .vertex_out(overlay_edit_flat_color_iface)
     .push_constant(Type::BOOL, "showCurveHandles")
-    .push_constant(Type::INT, "curveHandleDisplay")
+    .push_constant(Type::UINT, "curveHandleDisplay")
     .fragment_out(0, Type::VEC4, "fragColor")
     .vertex_source("overlay_edit_curve_point_vert.glsl")
     .fragment_source("overlay_point_varying_color_frag.glsl")
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
index 3a3c34b39e5..8d5febd6bb6 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
@@ -26,7 +26,7 @@ void main()
   uint color_id = (vert[1].flag >> COLOR_SHIFT);
 
   /* Don't output any edges if we don't show handles */
-  if (!showCurveHandles && (color_id < 5)) {
+  if (!showCurveHandles && (color_id < 5u)) {
     return;
   }
 
@@ -40,7 +40,7 @@ void main()
   if ((curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
     /* Nurbs must show the handles always. */
     bool is_u_segment = (((vert[1].flag ^ vert[0].flag) & EVEN_U_BIT) != 0u);
-    if ((!is_u_segment) && (color_id <= 4)) {
+    if ((!is_u_segment) && (color_id <= 4u)) {
       return;
     }
     if (is_gpencil) {
@@ -49,24 +49,24 @@ void main()
   }
 
   vec4 inner_color;
-  if (color_id == 0) {
+  if (color_id == 0u) {
     inner_color = (edge_selected) ? colorHandleSelFree : colorHandleFree;
   }
-  else if (color_id == 1) {
+  else if (color_id == 1u) {
     inner_color = (edge_selected) ? colorHandleSelAuto : colorHandleAuto;
   }
-  else if (color_id == 2) {
+  else if (color_id == 2u) {
     inner_color = (edge_selected) ? colorHandleSelVect : colorHandleVect;
   }
-  else if (color_id == 3) {
+  else if (color_id == 3u) {
     inner_color = (edge_selected) ? colorHandleSelAlign : colorHandleAlign;
   }
-  else if (color_id == 4) {
+  else if (color_id == 4u) {
     inner_color = (edge_selected) ? colorHandleSelAutoclamp : colorHandleAutoclamp;
   }
   else {
-    bool is_selected = (((vert[1].flag & vert[0].flag) & VERT_SELECTED) != 0);
-    bool is_u_segment = (((vert[1].flag ^ vert[0].flag) & EVEN_U_BIT) != 0);
+    bool is_selected = (((vert[1].flag & vert[0].flag) & VERT_SELECTED) != 0u);
+    bool is_u_segment = (((vert[1].flag ^ vert[0].flag) & EVEN_U_BIT) != 0u);
     if (is_u_segment) {
       inner_color = (is_selected) ? colorNurbSelUline : colorNurbUline;
     }
@@ -75,7 +75,7 @@ void main()
     }
   }
 
-  vec4 outer_color = (is_active_nurb != 0) ?
+  vec4 outer_color = (is_active_nurb != 0u) ?
                          mix(colorActiveSpline,
                              inner_color,
                              0.25) /* Minimize active color bleeding on inner_color. */
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
index 518b98e4ce5..24e61ae516d 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
@@ -47,7 +47,7 @@ void main()
   uint color_id = (vert_flag[1] >> COLOR_SHIFT);
 
   /* Don't output any edges if we don't show handles */
-  if (!showCurveHandles && (color_id < 5)) {
+  if (!showCurveHandles && (color_id < 5u)) {
     return;
   }
 
@@ -61,7 +61,7 @@ void main()
   if ((curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
     /* Nurbs must show the handles always. */
     bool is_u_segment = (((vert_flag[1] ^ vert_flag[0]) & EVEN_U_BIT) != 0u);
-    if ((!is_u_segment) && (color_id <= 4)) {
+    if ((!is_u_segment) && (color_id <= 4u)) {
       return;
     }
     if (is_gpencil) {
@@ -70,24 +70,24 @@ void main()
   }
 
   vec4 inner_color;
-  if (color_id == 0) {
+  if (color_id == 0u) {
     inner_color = (edge_selected) ? colorHandleSelFree : colorHandleFree;
   }
-  else if (color_id == 1) {
+  else if (color_id == 1u) {
     inner_color = (edge_selected) ? colorHandleSelAuto : colorHandleAuto;
   }
-  else if (color_id == 2) {
+  else if (color_id == 2u) {
     inner_color = (edge_selected) ? colorHandleSelVect : colorHandleVect;
   }
-  else if (color_id == 3) {
+  else if (color_id == 3u) {
     inner_color = (edge_selected) ? colorHandleSelAlign : colorHandleAlign;
   }
-  else if (color_id == 4) {
+  else if (color_id == 4u) {
     inner_color = (edge_selected) ? colorHandleSelAutoclamp : colorHandleAutoclamp;
   }
   else {
-    bool is_selected = (((vert_flag[1] & vert_flag[0]) & VERT_SELECTED) != 0);
-    bool is_u_segment = (((vert_flag[1] ^ vert_flag[0]) & EVEN_U_BIT) != 0);
+    bool is_selected = (((vert_flag[1] & vert_flag[0]) & VERT_SELECTED) != 0u);
+    bool is_u_segment = (((vert_flag[1] ^ vert_flag[0]) & EVEN_U_BIT) != 0u);
     if (is_u_segment) {
       inner_color = (is_selected) ? colorNurbSelUline : colorNurbUline;
     }
@@ -96,7 +96,7 @@ void main()
     }
   }
 
-  vec4 outer_color = (is_active_nurb != 0) ?
+  vec4 outer_color = (is_active_nurb != 0u) ?
                          mix(colorActiveSpline,
                              inner_color,
                              0.25) /* Minimize active color bleeding on inner_color. */
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
index a30496177c3..c6cf9f9b4f2 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
@@ -8,8 +8,8 @@ void main()
 
   /* Reuse the FREESTYLE flag to determine is GPencil. */
   bool is_gpencil = ((data & EDGE_FREESTYLE) != 0);
-  if ((data & VERT_SELECTED) != 0) {
-    if ((data & VERT_ACTIVE) != 0) {
+  if ((data & VERT_SELECTED) != 0u) {
+    if ((data & VERT_ACTIVE) != 0u) {
       finalColor = colorEditMeshActive;
     }
     else {
@@ -26,7 +26,7 @@ void main()
   view_clipping_distances(world_pos);
 
   bool show_handle = showCurveHandles;
-  if ((curveHandleDisplay == CURVE_HANDLE_SELECTED) && ((data & VERT_SELECTED_BEZT_HANDLE) == 0)) {
+  if ((curveHandleDisplay == CURVE_HANDLE_SELECTED) && ((data & VERT_SELECTED_BEZT_HANDLE) == 0u)) {
     show_handle = false;
   }
 
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_lattice_point_vert.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_lattice_point_vert.glsl
index 38ba80a981a..265fb818ec5 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_lattice_point_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_lattice_point_vert.glsl
@@ -6,10 +6,10 @@ void main()
 {
   GPU_INTEL_VERTEX_SHADER_WORKAROUND
 
-  if ((data & VERT_SELECTED) != 0) {
+  if ((data & VERT_SELECTED) != 0u) {
     finalColor = colorVertexSelect;
   }
-  else if ((data & VERT_ACTIVE) != 0) {
+  else if ((data & VERT_ACTIVE) != 0u) {
     finalColor = colorEditMeshActive;
   }
   else {
-- 
2.30.2


From 8ee4ba7b0ff98661f14752a4867f92ecc2110c31 Mon Sep 17 00:00:00 2001
From: Jeroen Bakker <jeroen@blender.org>
Date: Tue, 11 Apr 2023 11:38:15 +0200
Subject: [PATCH 2/4] Don't change interface of shaders.

---
 .../engines/overlay/shaders/infos/overlay_edit_mode_info.hh | 6 +++---
 .../overlay/shaders/overlay_edit_curve_handle_geom.glsl     | 2 +-
 .../shaders/overlay_edit_curve_handle_vert_no_geom.glsl     | 2 +-
 .../overlay/shaders/overlay_edit_curve_point_vert.glsl      | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh b/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh
index fe03952bbc8..cc6ae57a60e 100644
--- a/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh
+++ b/source/blender/draw/engines/overlay/shaders/infos/overlay_edit_mode_info.hh
@@ -396,7 +396,7 @@ GPU_SHADER_CREATE_INFO(overlay_edit_curve_handle)
     .geometry_layout(PrimitiveIn::LINES, PrimitiveOut::TRIANGLE_STRIP, 10)
     .geometry_out(overlay_edit_smooth_color_iface)
     .push_constant(Type::BOOL, "showCurveHandles")
-    .push_constant(Type::UINT, "curveHandleDisplay")
+    .push_constant(Type::INT, "curveHandleDisplay")
     .fragment_out(0, Type::VEC4, "fragColor")
     .vertex_source("overlay_edit_curve_handle_vert.glsl")
     .geometry_source("overlay_edit_curve_handle_geom.glsl")
@@ -414,7 +414,7 @@ GPU_SHADER_CREATE_INFO(overlay_edit_curve_handle_no_geom)
     .vertex_in(1, Type::UCHAR, "data")
     .vertex_out(overlay_edit_smooth_color_iface)
     .push_constant(Type::BOOL, "showCurveHandles")
-    .push_constant(Type::UINT, "curveHandleDisplay")
+    .push_constant(Type::INT, "curveHandleDisplay")
     .fragment_out(0, Type::VEC4, "fragColor")
     .vertex_source("overlay_edit_curve_handle_vert_no_geom.glsl")
     .fragment_source("overlay_varying_color.glsl")
@@ -439,7 +439,7 @@ GPU_SHADER_CREATE_INFO(overlay_edit_curve_point)
     .vertex_in(1, Type::UINT, "data")
     .vertex_out(overlay_edit_flat_color_iface)
     .push_constant(Type::BOOL, "showCurveHandles")
-    .push_constant(Type::UINT, "curveHandleDisplay")
+    .push_constant(Type::INT, "curveHandleDisplay")
     .fragment_out(0, Type::VEC4, "fragColor")
     .vertex_source("overlay_edit_curve_point_vert.glsl")
     .fragment_source("overlay_point_varying_color_frag.glsl")
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
index 8d5febd6bb6..b8cdb66ba32 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
@@ -37,7 +37,7 @@ void main()
   bool is_gpencil = ((vert[1].flag & VERT_GPENCIL_BEZT_HANDLE) != 0u);
 
   /* If handle type is only selected and the edge is not selected, don't show. */
-  if ((curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
+  if (uint(curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
     /* Nurbs must show the handles always. */
     bool is_u_segment = (((vert[1].flag ^ vert[0].flag) & EVEN_U_BIT) != 0u);
     if ((!is_u_segment) && (color_id <= 4u)) {
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
index 24e61ae516d..93f7eea2d1a 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
@@ -58,7 +58,7 @@ void main()
   bool is_gpencil = ((vert_flag[1] & VERT_GPENCIL_BEZT_HANDLE) != 0u);
 
   /* If handle type is only selected and the edge is not selected, don't show. */
-  if ((curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
+  if (uint(curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
     /* Nurbs must show the handles always. */
     bool is_u_segment = (((vert_flag[1] ^ vert_flag[0]) & EVEN_U_BIT) != 0u);
     if ((!is_u_segment) && (color_id <= 4u)) {
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
index c6cf9f9b4f2..be6ed2c1138 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
@@ -7,7 +7,7 @@ void main()
   GPU_INTEL_VERTEX_SHADER_WORKAROUND
 
   /* Reuse the FREESTYLE flag to determine is GPencil. */
-  bool is_gpencil = ((data & EDGE_FREESTYLE) != 0);
+  bool is_gpencil = ((data & EDGE_FREESTYLE) != 0u);
   if ((data & VERT_SELECTED) != 0u) {
     if ((data & VERT_ACTIVE) != 0u) {
       finalColor = colorEditMeshActive;
@@ -26,7 +26,7 @@ void main()
   view_clipping_distances(world_pos);
 
   bool show_handle = showCurveHandles;
-  if ((curveHandleDisplay == CURVE_HANDLE_SELECTED) && ((data & VERT_SELECTED_BEZT_HANDLE) == 0u)) {
+  if (uint(curveHandleDisplay == CURVE_HANDLE_SELECTED) && ((data & VERT_SELECTED_BEZT_HANDLE) == 0u)) {
     show_handle = false;
   }
 
-- 
2.30.2


From ab7d8e383f73124ab2a58b480d2ce3d8e587d388 Mon Sep 17 00:00:00 2001
From: Jeroen Bakker <jeroen@blender.org>
Date: Tue, 11 Apr 2023 11:39:42 +0200
Subject: [PATCH 3/4] Missing uint.

---
 .../engines/overlay/shaders/overlay_edit_curve_point_vert.glsl  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
index be6ed2c1138..a9d46ac5882 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
@@ -30,7 +30,7 @@ void main()
     show_handle = false;
   }
 
-  if (!show_handle && ((data & BEZIER_HANDLE) != 0)) {
+  if (!show_handle && ((data & BEZIER_HANDLE) != 0u)) {
     /* We set the vertex at the camera origin to generate 0 fragments. */
     gl_Position = vec4(0.0, 0.0, -3e36, 0.0);
   }
-- 
2.30.2


From 4dd1b6ea98164df8d4d4728e4a6d2b8f226c6121 Mon Sep 17 00:00:00 2001
From: Jeroen Bakker <jeroen@blender.org>
Date: Tue, 11 Apr 2023 12:32:04 +0200
Subject: [PATCH 4/4] Use correct fix.

---
 .../engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl | 2 +-
 .../overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl | 2 +-
 .../engines/overlay/shaders/overlay_edit_curve_point_vert.glsl  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
index b8cdb66ba32..5cd7f1c1325 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_geom.glsl
@@ -37,7 +37,7 @@ void main()
   bool is_gpencil = ((vert[1].flag & VERT_GPENCIL_BEZT_HANDLE) != 0u);
 
   /* If handle type is only selected and the edge is not selected, don't show. */
-  if (uint(curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
+  if ((uint(curveHandleDisplay) != CURVE_HANDLE_ALL) && (!handle_selected)) {
     /* Nurbs must show the handles always. */
     bool is_u_segment = (((vert[1].flag ^ vert[0].flag) & EVEN_U_BIT) != 0u);
     if ((!is_u_segment) && (color_id <= 4u)) {
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
index 93f7eea2d1a..5dc294e119f 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_handle_vert_no_geom.glsl
@@ -58,7 +58,7 @@ void main()
   bool is_gpencil = ((vert_flag[1] & VERT_GPENCIL_BEZT_HANDLE) != 0u);
 
   /* If handle type is only selected and the edge is not selected, don't show. */
-  if (uint(curveHandleDisplay != CURVE_HANDLE_ALL) && (!handle_selected)) {
+  if ((uint(curveHandleDisplay) != CURVE_HANDLE_ALL) && (!handle_selected)) {
     /* Nurbs must show the handles always. */
     bool is_u_segment = (((vert_flag[1] ^ vert_flag[0]) & EVEN_U_BIT) != 0u);
     if ((!is_u_segment) && (color_id <= 4u)) {
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
index a9d46ac5882..40cc7922949 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_edit_curve_point_vert.glsl
@@ -26,7 +26,7 @@ void main()
   view_clipping_distances(world_pos);
 
   bool show_handle = showCurveHandles;
-  if (uint(curveHandleDisplay == CURVE_HANDLE_SELECTED) && ((data & VERT_SELECTED_BEZT_HANDLE) == 0u)) {
+  if ((uint(curveHandleDisplay) == CURVE_HANDLE_SELECTED) && ((data & VERT_SELECTED_BEZT_HANDLE) == 0u)) {
     show_handle = false;
   }
 
-- 
2.30.2