BLI_math: ensure non-negative matrices for mat3_to_quat calculations
Making the callers responsible for this isn't practical as matrices are often passed indirectly to a functions such as mat3_to_axis_angle, BKE_object_mat3_to_rot & BKE_pchan_mat3_to_rot. Or the matrix is combined from other matrices which could be negative. Given quaternions calculated from negative matrices are completely invalid and checking only needs to negate matrices with a negative determinant, move the check into mat3_to_quat and related functions. Add mat3_normalized_to_quat_fast for cases no error checking on the input matrix is needed such as blending rotations.
This commit is contained in:
		| @@ -1243,19 +1243,12 @@ static PyObject *Matrix_to_quaternion(MatrixObject *self) | ||||
|                     "inappropriate matrix size - expects 3x3 or 4x4 matrix"); | ||||
|     return NULL; | ||||
|   } | ||||
|   float mat3[3][3]; | ||||
|   if (self->row_num == 3) { | ||||
|     copy_m3_m3(mat3, (const float(*)[3])self->matrix); | ||||
|     mat3_to_quat(quat, (const float(*)[3])self->matrix); | ||||
|   } | ||||
|   else { | ||||
|     copy_m3_m4(mat3, (const float(*)[4])self->matrix); | ||||
|     mat4_to_quat(quat, (const float(*)[4])self->matrix); | ||||
|   } | ||||
|   normalize_m3(mat3); | ||||
|   if (is_negative_m3(mat3)) { | ||||
|     /* Without this, the results are invalid, see: T94231. */ | ||||
|     negate_m3(mat3); | ||||
|   } | ||||
|   mat3_normalized_to_quat(quat, mat3); | ||||
|   return Quaternion_CreatePyObject(quat, NULL); | ||||
| } | ||||
|  | ||||
| @@ -1894,7 +1887,7 @@ static PyObject *Matrix_decompose(MatrixObject *self) | ||||
|   } | ||||
|  | ||||
|   mat4_to_loc_rot_size(loc, rot, size, (const float(*)[4])self->matrix); | ||||
|   mat3_to_quat(quat, rot); | ||||
|   mat3_normalized_to_quat_fast(quat, rot); | ||||
|  | ||||
|   ret = PyTuple_New(3); | ||||
|   PyTuple_SET_ITEMS(ret, | ||||
|   | ||||
		Reference in New Issue
	
	Block a user