Fix #33497: seting object scale to 0 on one axis made moving verts in edit mode

impossible. In this case matrix inversion failed and didn't give a useful result.
Now it falls back to a pseudoinverse in that case, so that moving along the
other axes still works.

There may be other places that can benefit from this, but this is a place where
it has no significant performance impact, doing this in general for e.g. bone
matrices could be quite slow.

source/blender/blenlib/BLI_math_matrix.h

@@ -144,10 +144,14 @@ float determinant_m2(float a, float b,
 float determinant_m3(float a, float b, float c,
                      float d, float e, float f,
                      float g, float h, float i);
+float determinant_m3_array(float m[3][3]);
 float determinant_m4(float A[4][4]);
 
+#define PSEUDOINVERSE_EPSILON 1e-8f
+
 void svd_m4(float U[4][4], float s[4], float V[4][4], float A[4][4]);
 void pseudoinverse_m4_m4(float Ainv[4][4], float A[4][4], float epsilon);
+void pseudoinverse_m3_m3(float Ainv[3][3], float A[3][3], float epsilon);
 
 /****************************** Transformations ******************************/
 

source/blender/blenlib/intern/math_matrix.c

@@ -504,8 +504,7 @@ void sub_m4_m4m4(float m1[4][4], float m2[4][4], float m3[4][4])
 			m1[i][j] = m2[i][j] - m3[i][j];
 }
 
-/* why not make this a standard part of the API? */
-static float determinant_m3_local(float m[3][3])
+float determinant_m3_array(float m[3][3])
 {
 	return (m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) -
 	        m[1][0] * (m[0][1] * m[2][2] - m[0][2] * m[2][1]) +
@@ -534,7 +533,7 @@ int invert_m3_m3_ex(float m1[3][3], float m2[3][3], const float epsilon)
 	adjoint_m3_m3(m1, m2);
 
 	/* then determinant old matrix! */
-	det = determinant_m3_local(m2);
+	det = determinant_m3_array(m2);
 
 	success = (fabsf(det) > epsilon);
 
@@ -569,7 +568,7 @@ int invert_m3_m3(float m1[3][3], float m2[3][3])
 	adjoint_m3_m3(m1, m2);
 
 	/* then determinant old matrix! */
-	det = determinant_m3_local(m2);
+	det = determinant_m3_array(m2);
 
 	success = (det != 0.0f);
 
@@ -1903,3 +1902,16 @@ void pseudoinverse_m4_m4(float Ainv[4][4], float A[4][4], float epsilon)
 
 	mul_serie_m4(Ainv, U, Wm, V, NULL, NULL, NULL, NULL, NULL);
 }
+
+void pseudoinverse_m3_m3(float Ainv[3][3], float A[3][3], float epsilon)
+{
+	/* try regular inverse when possible, otherwise fall back to slow svd */
+	if(!invert_m3_m3(Ainv, A)) {
+		float tmp[4][4], tmpinv[4][4];
+
+		copy_m4_m3(tmp, A);
+		pseudoinverse_m4_m4(tmpinv, tmp, epsilon);
+		copy_m3_m4(Ainv, tmpinv);
+	}
+}
+

source/blender/editors/transform/transform_conversions.c

@@ -272,7 +272,7 @@ static void createTransTexspace(TransInfo *t)
 	copy_m3_m4(td->mtx, ob->obmat);
 	copy_m3_m4(td->axismtx, ob->obmat);
 	normalize_m3(td->axismtx);
-	invert_m3_m3(td->smtx, td->mtx);
+	pseudoinverse_m3_m3(td->smtx, td->mtx, PSEUDOINVERSE_EPSILON);
 
 	if (BKE_object_obdata_texspace_get(ob, &texflag, &td->loc, &td->ext->size, &td->ext->rot)) {
 		ob->dtx |= OB_TEXSPACE;
@@ -316,7 +316,7 @@ static void createTransEdge(TransInfo *t)
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransCrease");
 
 	copy_m3_m4(mtx, t->obedit->obmat);
-	invert_m3_m3(smtx, mtx);
+	pseudoinverse_m3_m3(smtx, mtx, PSEUDOINVERSE_EPSILON);
 
 	BM_ITER_MESH (eed, &iter, em->bm, BM_EDGES_OF_MESH) {
 		if (!BM_elem_flag_test(eed, BM_ELEM_HIDDEN) && (BM_elem_flag_test(eed, BM_ELEM_SELECT) || propmode)) {
@@ -553,7 +553,7 @@ static void add_pose_transdata(TransInfo *t, bPoseChannel *pchan, Object *ob, Tr
 		invert_m3_m3(td->ext->r_smtx, td->ext->r_mtx);
 	}
 
-	invert_m3_m3(td->smtx, td->mtx);
+	pseudoinverse_m3_m3(td->smtx, td->mtx, PSEUDOINVERSE_EPSILON);
 
 	/* exceptional case: rotate the pose bone which also applies transformation
 	 * when a parentless bone has BONE_NO_LOCAL_LOCATION [] */
@@ -605,7 +605,7 @@ static void add_pose_transdata(TransInfo *t, bPoseChannel *pchan, Object *ob, Tr
 
 			/* only object matrix correction */
 			copy_m3_m3(td->mtx, omat);
-			invert_m3_m3(td->smtx, td->mtx);
+			pseudoinverse_m3_m3(td->smtx, td->mtx, PSEUDOINVERSE_EPSILON);
 		}
 	}
 
@@ -1054,7 +1054,7 @@ static void createTransArmatureVerts(TransInfo *t)
 	if (!t->total) return;
 
 	copy_m3_m4(mtx, t->obedit->obmat);
-	invert_m3_m3(smtx, mtx);
+	pseudoinverse_m3_m3(smtx, mtx, PSEUDOINVERSE_EPSILON);
 
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransEditBone");
 
@@ -1222,7 +1222,7 @@ static void createTransMBallVerts(TransInfo *t)
 	tx = t->ext = MEM_callocN(t->total * sizeof(TransDataExtension), "MetaElement_TransExtension");
 
 	copy_m3_m4(mtx, t->obedit->obmat);
-	invert_m3_m3(smtx, mtx);
+	pseudoinverse_m3_m3(smtx, mtx, PSEUDOINVERSE_EPSILON);
 
 	for (ml = mb->editelems->first; ml; ml = ml->next) {
 		if (propmode || (ml->flag & SELECT)) {
@@ -1379,7 +1379,7 @@ static void createTransCurveVerts(TransInfo *t)
 	t->data = MEM_callocN(t->total * sizeof(TransData), "TransObData(Curve EditMode)");
 
 	copy_m3_m4(mtx, t->obedit->obmat);
-	invert_m3_m3(smtx, mtx);
+	pseudoinverse_m3_m3(smtx, mtx, PSEUDOINVERSE_EPSILON);
 
 	td = t->data;
 	for (nu = nurbs->first; nu; nu = nu->next) {
@@ -1570,7 +1570,7 @@ static void createTransLatticeVerts(TransInfo *t)
 	t->data = MEM_callocN(t->total * sizeof(TransData), "TransObData(Lattice EditMode)");
 
 	copy_m3_m4(mtx, t->obedit->obmat);
-	invert_m3_m3(smtx, mtx);
+	pseudoinverse_m3_m3(smtx, mtx, PSEUDOINVERSE_EPSILON);
 
 	td = t->data;
 	bp = latt->def;
@@ -2052,7 +2052,9 @@ static void createTransEditVerts(TransInfo *t)
 	}
 
 	copy_m3_m4(mtx, t->obedit->obmat);
-	invert_m3_m3(smtx, mtx);
+	/* we use a pseudoinverse so that when one of the axes is scaled to 0,
+	 * matrix inversion still works and we can still moving along the other */
+	pseudoinverse_m3_m3(smtx, mtx, PSEUDOINVERSE_EPSILON);
 
 	if (propmode & T_PROP_CONNECTED) {
 		editmesh_set_connectivity_distance(em, mtx, dists);