Fix bad pointer cast when modifier is used on non-mesh object

This commit is contained in:
2018-05-09 17:37:54 +02:00
parent b726ff4fc4
commit 4fe5a105f6
8 changed files with 61 additions and 67 deletions

View File

@@ -135,11 +135,7 @@ static void deformVertsEM(
Mesh *mesh, float (*vertexCos)[3], int numVerts) Mesh *mesh, float (*vertexCos)[3], int numVerts)
{ {
ArmatureModifierData *amd = (ArmatureModifierData *) md; ArmatureModifierData *amd = (ArmatureModifierData *) md;
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, em, mesh, NULL, false, false);
if (!mesh) {
mesh_src = BKE_bmesh_to_mesh_nomain(em->bm, &(struct BMeshToMeshParams){0});
}
modifier_vgroup_cache(md, vertexCos); /* if next modifier needs original vertices */ modifier_vgroup_cache(md, vertexCos); /* if next modifier needs original vertices */
@@ -152,7 +148,7 @@ static void deformVertsEM(
amd->prevCos = NULL; amd->prevCos = NULL;
} }
if (!mesh) { if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src); BKE_id_free(NULL, mesh_src);
} }
} }
@@ -163,16 +159,12 @@ static void deformMatricesEM(
float (*defMats)[3][3], int numVerts) float (*defMats)[3][3], int numVerts)
{ {
ArmatureModifierData *amd = (ArmatureModifierData *) md; ArmatureModifierData *amd = (ArmatureModifierData *) md;
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, em, mesh, NULL, false, false);
if (!mesh) {
mesh_src = BKE_bmesh_to_mesh_nomain(em->bm, &(struct BMeshToMeshParams){0});
}
armature_deform_verts(amd->object, ctx->object, mesh_src, vertexCos, defMats, numVerts, armature_deform_verts(amd->object, ctx->object, mesh_src, vertexCos, defMats, numVerts,
amd->deformflag, NULL, amd->defgrp_name); amd->deformflag, NULL, amd->defgrp_name);
if (!mesh) { if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src); BKE_id_free(NULL, mesh_src);
} }
} }
@@ -181,10 +173,14 @@ static void deformMatrices(ModifierData *md, const ModifierEvalContext *ctx, Mes
float (*vertexCos)[3], float (*defMats)[3][3], int numVerts) float (*vertexCos)[3], float (*defMats)[3][3], int numVerts)
{ {
ArmatureModifierData *amd = (ArmatureModifierData *) md; ArmatureModifierData *amd = (ArmatureModifierData *) md;
Mesh *mesh_src = mesh ? mesh : ctx->object->data; Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
armature_deform_verts(amd->object, ctx->object, mesh_src, vertexCos, defMats, numVerts, armature_deform_verts(amd->object, ctx->object, mesh_src, vertexCos, defMats, numVerts,
amd->deformflag, NULL, amd->defgrp_name); amd->deformflag, NULL, amd->defgrp_name);
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }
ModifierTypeInfo modifierType_Armature = { ModifierTypeInfo modifierType_Armature = {

View File

@@ -428,12 +428,7 @@ static void deformVerts(ModifierData *md, const ModifierEvalContext *ctx,
int numVerts) int numVerts)
{ {
CastModifierData *cmd = (CastModifierData *)md; CastModifierData *cmd = (CastModifierData *)md;
Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
Mesh *mesh_src = mesh;
if (mesh_src == NULL) {
mesh_src = ctx->object->data;
}
BLI_assert(mesh_src->totvert == numVerts); BLI_assert(mesh_src->totvert == numVerts);
@@ -443,6 +438,10 @@ static void deformVerts(ModifierData *md, const ModifierEvalContext *ctx,
else { /* MOD_CAST_TYPE_SPHERE or MOD_CAST_TYPE_CYLINDER */ else { /* MOD_CAST_TYPE_SPHERE or MOD_CAST_TYPE_CYLINDER */
sphere_do(cmd, ctx->object, mesh_src, vertexCos, numVerts); sphere_do(cmd, ctx->object, mesh_src, vertexCos, numVerts);
} }
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }
static void deformVertsEM( static void deformVertsEM(
@@ -451,12 +450,7 @@ static void deformVertsEM(
Mesh *mesh, float (*vertexCos)[3], int numVerts) Mesh *mesh, float (*vertexCos)[3], int numVerts)
{ {
CastModifierData *cmd = (CastModifierData *)md; CastModifierData *cmd = (CastModifierData *)md;
Mesh *mesh_src = get_mesh(ctx->object, editData, mesh, NULL, false, false);
Mesh *mesh_src = mesh;
if (mesh_src == NULL) {
mesh_src = BKE_bmesh_to_mesh_nomain(editData->bm, &(struct BMeshToMeshParams){0});
}
BLI_assert(mesh_src->totvert == numVerts); BLI_assert(mesh_src->totvert == numVerts);
@@ -467,7 +461,7 @@ static void deformVertsEM(
sphere_do(cmd, ctx->object, mesh_src, vertexCos, numVerts); sphere_do(cmd, ctx->object, mesh_src, vertexCos, numVerts);
} }
if (!mesh) { if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src); BKE_id_free(NULL, mesh_src);
} }
} }

View File

@@ -51,6 +51,7 @@
#include "DEG_depsgraph_build.h" #include "DEG_depsgraph_build.h"
#include "MOD_modifiertypes.h" #include "MOD_modifiertypes.h"
#include "MOD_util.h"
static void initData(ModifierData *md) static void initData(ModifierData *md)
{ {
@@ -112,18 +113,17 @@ static void deformVerts(
int numVerts) int numVerts)
{ {
CurveModifierData *cmd = (CurveModifierData *) md; CurveModifierData *cmd = (CurveModifierData *) md;
Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
Mesh *mesh_src = mesh;
if (mesh_src == NULL) {
mesh_src = ctx->object->data;
}
BLI_assert(mesh_src->totvert == numVerts); BLI_assert(mesh_src->totvert == numVerts);
/* silly that defaxis and curve_deform_verts are off by 1 /* silly that defaxis and curve_deform_verts are off by 1
* but leave for now to save having to call do_versions */ * but leave for now to save having to call do_versions */
curve_deform_verts(cmd->object, ctx->object, mesh_src, vertexCos, numVerts, cmd->name, cmd->defaxis - 1); curve_deform_verts(cmd->object, ctx->object, mesh_src, vertexCos, numVerts, cmd->name, cmd->defaxis - 1);
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }
static void deformVertsEM( static void deformVertsEM(
@@ -134,17 +134,13 @@ static void deformVertsEM(
float (*vertexCos)[3], float (*vertexCos)[3],
int numVerts) int numVerts)
{ {
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, em, mesh, NULL, false, false);
if (mesh_src == NULL) {
mesh_src = BKE_bmesh_to_mesh_nomain(em->bm, &(struct BMeshToMeshParams){0});
}
BLI_assert(mesh_src->totvert == numVerts); BLI_assert(mesh_src->totvert == numVerts);
deformVerts(md, ctx, mesh_src, vertexCos, numVerts); deformVerts(md, ctx, mesh_src, vertexCos, numVerts);
if (!mesh) { if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src); BKE_id_free(NULL, mesh_src);
} }
} }

View File

@@ -367,33 +367,29 @@ static void deformVerts(
float (*vertexCos)[3], float (*vertexCos)[3],
int numVerts) int numVerts)
{ {
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
if (mesh_src == NULL) {
mesh_src = ctx->object->data;
}
BLI_assert(mesh_src->totvert == numVerts); BLI_assert(mesh_src->totvert == numVerts);
displaceModifier_do((DisplaceModifierData *)md, ctx->object, mesh_src, displaceModifier_do((DisplaceModifierData *)md, ctx->object, mesh_src,
vertexCos, numVerts); vertexCos, numVerts);
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }
static void deformVertsEM( static void deformVertsEM(
ModifierData *md, const ModifierEvalContext *ctx, struct BMEditMesh *editData, ModifierData *md, const ModifierEvalContext *ctx, struct BMEditMesh *editData,
Mesh *mesh, float (*vertexCos)[3], int numVerts) Mesh *mesh, float (*vertexCos)[3], int numVerts)
{ {
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, editData, mesh, NULL, false, false);
if (mesh_src == NULL) {
mesh_src = BKE_bmesh_to_mesh_nomain(editData->bm, &(struct BMeshToMeshParams){0});
}
BLI_assert(mesh_src->totvert == numVerts); BLI_assert(mesh_src->totvert == numVerts);
displaceModifier_do((DisplaceModifierData *)md, ctx->object, mesh_src, vertexCos, numVerts); displaceModifier_do((DisplaceModifierData *)md, ctx->object, mesh_src, vertexCos, numVerts);
if (!mesh) { if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src); BKE_id_free(NULL, mesh_src);
} }
} }

View File

@@ -352,9 +352,13 @@ static void deformVerts(struct ModifierData *md, const struct ModifierEvalContex
float (*vertexCos)[3], int numVerts) float (*vertexCos)[3], int numVerts)
{ {
HookModifierData *hmd = (HookModifierData *)md; HookModifierData *hmd = (HookModifierData *)md;
Mesh *mesh_src = mesh ? mesh : ctx->object->data; Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
deformVerts_do(hmd, ctx->object, mesh_src, vertexCos, numVerts); deformVerts_do(hmd, ctx->object, mesh_src, vertexCos, numVerts);
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }
static void deformVertsEM(struct ModifierData *md, const struct ModifierEvalContext *ctx, static void deformVertsEM(struct ModifierData *md, const struct ModifierEvalContext *ctx,
@@ -362,10 +366,7 @@ static void deformVertsEM(struct ModifierData *md, const struct ModifierEvalCont
struct Mesh *mesh, float (*vertexCos)[3], int numVerts) struct Mesh *mesh, float (*vertexCos)[3], int numVerts)
{ {
HookModifierData *hmd = (HookModifierData *)md; HookModifierData *hmd = (HookModifierData *)md;
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, editData, mesh, NULL, false, false);
if (!mesh) {
mesh_src = BKE_bmesh_to_mesh_nomain(editData->bm, &(struct BMeshToMeshParams){0});
}
deformVerts_do(hmd, ctx->object, mesh_src, vertexCos, numVerts); deformVerts_do(hmd, ctx->object, mesh_src, vertexCos, numVerts);

View File

@@ -99,27 +99,26 @@ static void deformVerts(ModifierData *md, const ModifierEvalContext *ctx,
int numVerts) int numVerts)
{ {
LatticeModifierData *lmd = (LatticeModifierData *) md; LatticeModifierData *lmd = (LatticeModifierData *) md;
struct Mesh *mesh_src = mesh ? mesh : ctx->object->data; struct Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
modifier_vgroup_cache(md, vertexCos); /* if next modifier needs original vertices */ modifier_vgroup_cache(md, vertexCos); /* if next modifier needs original vertices */
lattice_deform_verts(lmd->object, ctx->object, mesh_src, lattice_deform_verts(lmd->object, ctx->object, mesh_src,
vertexCos, numVerts, lmd->name, lmd->strength); vertexCos, numVerts, lmd->name, lmd->strength);
}
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}}
static void deformVertsEM( static void deformVertsEM(
ModifierData *md, const ModifierEvalContext *ctx, struct BMEditMesh *em, ModifierData *md, const ModifierEvalContext *ctx, struct BMEditMesh *em,
struct Mesh *mesh, float (*vertexCos)[3], int numVerts) struct Mesh *mesh, float (*vertexCos)[3], int numVerts)
{ {
struct Mesh *mesh_src = mesh; struct Mesh *mesh_src = get_mesh(ctx->object, em, mesh, NULL, false, false);
if (!mesh) {
mesh_src = BKE_bmesh_to_mesh_nomain(em->bm, &(struct BMeshToMeshParams){0});
}
deformVerts(md, ctx, mesh_src, vertexCos, numVerts); deformVerts(md, ctx, mesh_src, vertexCos, numVerts);
if (!mesh) { if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src); BKE_id_free(NULL, mesh_src);
} }
} }

View File

@@ -42,6 +42,7 @@
#include "BKE_cdderivedmesh.h" #include "BKE_cdderivedmesh.h"
#include "BKE_editmesh.h" #include "BKE_editmesh.h"
#include "BKE_mesh.h" #include "BKE_mesh.h"
#include "BKE_library.h"
#include "BKE_library_query.h" #include "BKE_library_query.h"
#include "BKE_modifier.h" #include "BKE_modifier.h"
#include "BKE_deform.h" #include "BKE_deform.h"
@@ -383,8 +384,13 @@ static void deformVerts(ModifierData *md, const ModifierEvalContext *ctx,
float (*vertexCos)[3], float (*vertexCos)[3],
int numVerts) int numVerts)
{ {
Mesh *mesh_src = mesh ? mesh : ctx->object->data; Mesh *mesh_src = get_mesh(ctx->object, NULL, mesh, NULL, false, false);
SimpleDeformModifier_do((SimpleDeformModifierData *)md, ctx->object, mesh_src, vertexCos, numVerts); SimpleDeformModifier_do((SimpleDeformModifierData *)md, ctx->object, mesh_src, vertexCos, numVerts);
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }
static void deformVertsEM(ModifierData *md, const ModifierEvalContext *ctx, static void deformVertsEM(ModifierData *md, const ModifierEvalContext *ctx,
@@ -393,11 +399,13 @@ static void deformVertsEM(ModifierData *md, const ModifierEvalContext *ctx,
float (*vertexCos)[3], float (*vertexCos)[3],
int numVerts) int numVerts)
{ {
Mesh *mesh_src = mesh; Mesh *mesh_src = get_mesh(ctx->object, editData, mesh, NULL, false, false);
if (!mesh) {
mesh_src = BKE_bmesh_to_mesh_nomain(editData->bm, &(struct BMeshToMeshParams){0});
}
SimpleDeformModifier_do((SimpleDeformModifierData *)md, ctx->object, mesh_src, vertexCos, numVerts); SimpleDeformModifier_do((SimpleDeformModifierData *)md, ctx->object, mesh_src, vertexCos, numVerts);
if (mesh_src != mesh) {
BKE_id_free(NULL, mesh_src);
}
} }

View File

@@ -300,6 +300,8 @@ Mesh *get_mesh(Object *ob, struct BMEditMesh *em, Mesh *mesh,
struct BMeshToMeshParams bmtmp = {0}; struct BMeshToMeshParams bmtmp = {0};
if (em) mesh = BKE_bmesh_to_mesh_nomain(em->bm, &bmtmp); if (em) mesh = BKE_bmesh_to_mesh_nomain(em->bm, &bmtmp);
else { else {
/* TODO(sybren): after modifier conversion of DM to Mesh is done, check whether
* we really need a copy here. Maybe the CoW ob->data can be directly used. */
BKE_id_copy_ex( BKE_id_copy_ex(
NULL, ob->data, (ID **)&mesh, NULL, ob->data, (ID **)&mesh,
LIB_ID_CREATE_NO_MAIN | LIB_ID_CREATE_NO_MAIN |
@@ -308,6 +310,8 @@ Mesh *get_mesh(Object *ob, struct BMEditMesh *em, Mesh *mesh,
false); false);
} }
/* TODO(sybren): after modifier conversion of DM to Mesh is done, check whether
* we really need vertexCos here. */
if (vertexCos) { if (vertexCos) {
BKE_mesh_apply_vert_coords(mesh, vertexCos); BKE_mesh_apply_vert_coords(mesh, vertexCos);
mesh->runtime.cd_dirty_vert |= CD_MASK_NORMAL; mesh->runtime.cd_dirty_vert |= CD_MASK_NORMAL;