source/blender/blenlib/BLI_math_vector.hh

@@ -665,6 +665,26 @@ template<typename T, int Size>
   return true;
 }
 
+/**
+ * Return the distance from \a point to the line that goes through \a a and \a b.
+ */
+template<typename T>
+[[nodiscard]] inline T distance_to_line(const VecBase<T, 3> &point,
+                                        const VecBase<T, 3> &a,
+                                        const VecBase<T, 3> &b)
+{
+  const VecBase<T, 3> a_b = b - a;
+  const VecBase<T, 3> a_point = point - a;
+  T length_a_b;
+  T length_a_point;
+  const VecBase<T, 3> a_b_norm = normalize_and_get_length(a_b, length_a_b);
+  const VecBase<T, 3> a_point_norm = normalize_and_get_length(a_point, length_a_point);
+  const float sin_angle_a = length(cross(a_b_norm, a_point_norm));
+  /* Using the law of sines, sin(90) is 1 so the division isn't needed. */
+  const T distance = length_a_point * sin_angle_a;
+  return distance;
+}
+
 /** Intersections. */
 
 template<typename T> struct isect_result {

source/blender/blenlib/tests/BLI_math_vector_test.cc

@@ -181,4 +181,28 @@ TEST(math_vector, exp)
   EXPECT_NEAR(result.z, 20.085536923187668f, 1e-6f);
 }
 
+TEST(math_vector, distance_to_line)
+{
+  float3 p1(1.0f, 1.0f, 0.0f);
+  const float3 a1(0, 0, 0);
+  const float3 b1(1, 0, 0);
+
+  float result = math::distance_to_line(p1, a1, b1);
+  EXPECT_NEAR(result, 1.0f, 1e-6f);
+
+  float3 p2(-1.0f, 2.0f, 0.0f);
+  result = math::distance_to_line(p2, a1, b1);
+  EXPECT_NEAR(result, 2.0f, 1e-6f);
+
+  float3 p3(-1.0f, 0.0f, 0.0f);
+  result = math::distance_to_line(p3, a1, b1);
+  EXPECT_NEAR(result, 0.0f, 1e-6f);
+
+  float3 p4(-1.0f, 8.0f, -1.0f);
+  const float3 a2(-1, 0, -1);
+  const float3 b2(1, 0, 1);
+  result = math::distance_to_line(p4, a2, b2);
+  EXPECT_NEAR(result, 8.0f, 1e-6f);
+}
+
 }  // namespace blender::tests

source/blender/editors/mesh/editface.cc

@@ -361,19 +361,23 @@ static int find_closest_edge_in_poly(ARegion *region,
   using namespace blender;
   int closest_edge_index;
 
-  const float2 mval_f = {float(mval[0]), float(mval[1])};
+  const float3 mval_f = {float(mval[0]), float(mval[1]), 0};
   float min_distance = FLT_MAX;
   for (const int i : poly_edges) {
-    float2 screen_coordinate;
     const int2 edge = edges[i];
-    const float3 edge_vert_average = math::midpoint(vert_positions[edge[0]],
-                                                    vert_positions[edge[1]]);
-    eV3DProjStatus status = ED_view3d_project_float_object(
-        region, edge_vert_average, screen_coordinate, V3D_PROJ_TEST_CLIP_DEFAULT);
-    if (status != V3D_PROJ_RET_OK) {
+
+    float3 screen_coordinate_0;
+    float3 screen_coordinate_1;
+    eV3DProjStatus status_vert_0 = ED_view3d_project_float_object(
+        region, vert_positions[edge[0]], screen_coordinate_0, V3D_PROJ_TEST_CLIP_DEFAULT);
+    eV3DProjStatus status_vert_1 = ED_view3d_project_float_object(
+        region, vert_positions[edge[1]], screen_coordinate_1, V3D_PROJ_TEST_CLIP_DEFAULT);
+    /* This means the edge can only be checked for if both verts are on the screen. */
+    if (status_vert_0 != V3D_PROJ_RET_OK || status_vert_1 != V3D_PROJ_RET_OK) {
       continue;
     }
-    const float distance = math::distance_squared(mval_f, screen_coordinate);
+    const float distance = math::distance_to_line(
+        mval_f, screen_coordinate_0, screen_coordinate_1);
     if (distance < min_distance) {
       min_distance = distance;
       closest_edge_index = i;