Dupli Bugfixes!

It seems everytime I try to fix something here I break something
else, but anyway, another try at getting this to work properly.

Fixes for:
- Getting dupliverts/faces derivedmesh with orco caused wrong results
  on meshes in linked dupligroups with proxy, because modifier stack
  was revaluated with wrong object matrix, now gets orco another way.
- Fix render instances being hidden when original object was not added
  to the object render list.
- Changed the way object instances find their original objects, now
  works the other way around, original objects look for their instances
  instead.

There's probably issues still with recursive dupligroups..

source/blender/blenkernel/intern/anim.c

@@ -65,6 +65,7 @@
 #include "BKE_key.h"
 #include "BKE_lattice.h"
 #include "BKE_main.h"
+#include "BKE_mesh.h"
 #include "BKE_object.h"
 #include "BKE_particle.h"
 #include "BKE_utildefines.h"
@@ -423,6 +424,7 @@ static void vertex_dupli__mapFunc(void *userData, int index, float *co, float *n
 static void vertex_duplilist(ListBase *lb, ID *id, Object *par, float par_space_mat[][4], int level)
 {
 	Object *ob, *ob_iter;
+	Mesh *me;
 	Base *base = NULL;
 	float vec[3], no[3], pmat[4][4];
 	int lay, totvert, a, oblay;
@@ -431,23 +433,22 @@ static void vertex_duplilist(ListBase *lb, ID *id, Object *par, float par_space_
 	Scene *sce = NULL;
 	Group *group = NULL;
 	GroupObject * go = NULL;
-	CustomDataMask dataMask = CD_MASK_BAREMESH;
 	
 	Mat4CpyMat4(pmat, par->obmat);
 	
 	/* simple preventing of too deep nested groups */
 	if(level>MAX_DUPLI_RECUR) return;
 
-	if(G.rendering)
-		dataMask |= CD_MASK_ORCO;
-	
 	if(par==G.obedit)
-		dm= editmesh_get_derived_cage(dataMask);
+		dm= editmesh_get_derived_cage(CD_MASK_BAREMESH);
 	else
-		dm= mesh_get_derived_deform(par, dataMask);
+		dm= mesh_get_derived_deform(par, CD_MASK_BAREMESH);
 
-	if(G.rendering)
-		vdd.orco= dm->getVertDataArray(dm, CD_ORCO);
+	if(G.rendering) {
+		me= par->data;
+		vdd.orco= (float(*)[3])get_mesh_orco_verts(par);
+		transform_mesh_orco_verts(me, vdd.orco, me->totvert, 0);
+	}
 	else
 		vdd.orco= NULL;
 	
@@ -521,6 +522,8 @@ static void vertex_duplilist(ListBase *lb, ID *id, Object *par, float par_space_
 		else		go= go->next;		/* group loop */
 	}
 
+	if(vdd.orco)
+		MEM_freeN(vdd.orco);
 	dm->release(dm);
 }
 
@@ -534,25 +537,21 @@ static void face_duplilist(ListBase *lb, ID *id, Object *par, float par_space_ma
 	MTFace *mtface;
 	MFace *mface;
 	MVert *mvert;
-	float pmat[4][4], imat[3][3], (*orco)[3], w;
+	float pmat[4][4], imat[3][3], (*orco)[3] = NULL, w;
 	int lay, oblay, totface, a;
 	Scene *sce = NULL;
 	Group *group = NULL;
 	GroupObject *go = NULL;
-	CustomDataMask dataMask = CD_MASK_BAREMESH;
 	float ob__obmat[4][4]; /* needed for groups where the object matrix needs to be modified */
 	
 	/* simple preventing of too deep nested groups */
 	if(level>MAX_DUPLI_RECUR) return;
 	
-	if(G.rendering)
-		dataMask |= CD_MASK_ORCO;
-	
 	Mat4CpyMat4(pmat, par->obmat);
 
 	if(par==G.obedit) {
 		int totvert;
-		dm= editmesh_get_derived_cage(dataMask);
+		dm= editmesh_get_derived_cage(CD_MASK_BAREMESH);
 		
 		totface= dm->getNumFaces(dm);
 		mface= MEM_mallocN(sizeof(MFace)*totface, "mface temp");
@@ -562,7 +561,7 @@ static void face_duplilist(ListBase *lb, ID *id, Object *par, float par_space_ma
 		dm->copyVertArray(dm, mvert);
 	}
 	else {
-		dm = mesh_get_derived_deform(par, dataMask);
+		dm = mesh_get_derived_deform(par, CD_MASK_BAREMESH);
 		
 		totface= dm->getNumFaces(dm);
 		mface= dm->getFaceArray(dm);
@@ -571,7 +570,9 @@ static void face_duplilist(ListBase *lb, ID *id, Object *par, float par_space_ma
 
 	if(G.rendering) {
 		me= (Mesh*)par->data;
-		orco= dm->getVertDataArray(dm, CD_ORCO);
+
+		orco= (float(*)[3])get_mesh_orco_verts(par);
+		transform_mesh_orco_verts(me, orco, me->totvert, 0);
 		mtface= me->mtface;
 	}
 	else {
@@ -711,6 +712,9 @@ static void face_duplilist(ListBase *lb, ID *id, Object *par, float par_space_ma
 		MEM_freeN(mface);
 		MEM_freeN(mvert);
 	}
+
+	if(orco)
+		MEM_freeN(orco);
 	
 	dm->release(dm);
 }

source/blender/render/intern/include/render_types.h

@@ -174,7 +174,6 @@ struct Render
 
 	struct ObjectInstanceRen *objectinstance;
 	ListBase instancetable;
-	struct GHash *objecthash;
 	int totinstance;
 
 	struct Image *backbuf, *bakebuf;

source/blender/render/intern/source/convertblender.c

@@ -3870,6 +3870,29 @@ static int render_object_type(int type)
 	return ELEM5(type, OB_FONT, OB_CURVE, OB_SURF, OB_MESH, OB_MBALL);
 }
 
+static void find_dupli_instances(Render *re, ObjectRen *obr)
+{
+	ObjectInstanceRen *obi;
+	float imat[4][4], obmat[4][4], obimat[4][4], nmat[3][3];
+
+	Mat4MulMat4(obmat, obr->ob->obmat, re->viewmat);
+	Mat4Invert(imat, obmat);
+
+	for(obi=re->instancetable.last; obi; obi=obi->prev) {
+		if(!obi->obr && obi->ob == obr->ob && obi->psysindex == obr->psysindex) {
+			obi->obr= obr;
+
+			/* compute difference between object matrix and
+			 * object matrix with dupli transform, in viewspace */
+			Mat4CpyMat4(obimat, obi->mat);
+			Mat4MulMat4(obi->mat, imat, obimat);
+
+			Mat3CpyMat4(nmat, obi->mat);
+			Mat3Inv(obi->imat, nmat);
+		}
+	}
+}
+
 static void init_render_object_data(Render *re, ObjectRen *obr, int only_verts)
 {
 	Object *ob= obr->ob;
@@ -3937,6 +3960,8 @@ static void add_render_object(Render *re, Object *ob, Object *par, int index, in
 		/* only add instance for objects that have not been used for dupli */
 		if(!(ob->transflag & OB_RENDER_DUPLI))
 			RE_addRenderInstance(re, obr, ob, par, index, 0, NULL);
+		else
+			find_dupli_instances(re, obr);
 	}
 
 	/* and one render object per particle system */
@@ -3950,6 +3975,8 @@ static void add_render_object(Render *re, Object *ob, Object *par, int index, in
 			/* only add instance for objects that have not been used for dupli */
 			if(!(ob->transflag & OB_RENDER_DUPLI))
 				RE_addRenderInstance(re, obr, ob, par, index, psysindex, NULL);
+			else
+				find_dupli_instances(re, obr);
 		}
 	}
 }
@@ -4137,7 +4164,7 @@ static void database_init_objects(Render *re, unsigned int lay, int nolamps, int
 	Object *ob;
 	ObjectInstanceRen *obi;
 	Scene *sce;
-	float mat[4][4], obmat[4][4];
+	float mat[4][4];
 
 	for(SETLOOPER(re->scene, base)) {
 		ob= base->object;
@@ -4173,11 +4200,10 @@ static void database_init_objects(Render *re, unsigned int lay, int nolamps, int
 				for(dob= lb->first; dob; dob= dob->next) {
 					Object *obd= dob->ob;
 					
-					Mat4CpyMat4(obmat, obd->obmat);
 					Mat4CpyMat4(obd->obmat, dob->mat);
 
 					/* group duplis need to set ob matrices correct, for deform. so no_draw is part handled */
-					if(dob->no_draw)
+					if(!(obd->transflag & OB_RENDER_DUPLI) && dob->no_draw)
 						continue;
 
 					if(obd->restrictflag & OB_RESTRICT_RENDER)
@@ -4192,13 +4218,9 @@ static void database_init_objects(Render *re, unsigned int lay, int nolamps, int
 					if(allow_render_dupli_instance(re, dob, obd)) {
 						ParticleSystem *psys;
 						int psysindex;
-						float imat[4][4], mat[4][4];
-
-						/* compute difference between object matrix and
-						 * object matrix with dupli transform, in viewspace */
-						Mat4Invert(imat, obmat);
-						MTC_Mat4MulSerie(mat, re->viewmat, dob->mat, imat, re->viewinv, 0, 0, 0, 0);
+						float mat[4][4];
 
+						Mat4MulMat4(mat, dob->mat, re->viewmat);
 						obi= RE_addRenderInstance(re, NULL, obd, ob, dob->index, 0, mat);
 						VECCOPY(obi->dupliorco, dob->orco);
 						obi->dupliuv[0]= dob->uv[0];
@@ -4214,8 +4236,6 @@ static void database_init_objects(Render *re, unsigned int lay, int nolamps, int
 						
 						obd->flag |= OB_DONE;
 						obd->transflag |= OB_RENDER_DUPLI;
-
-						Mat4CpyMat4(obd->obmat, obmat);
 					}
 					else
 						init_render_object(re, obd, ob, dob->index, only_verts);

source/blender/render/intern/source/renderdatabase.c

@@ -408,6 +408,7 @@ int RE_vlakren_get_normal(Render *re, ObjectInstanceRen *obi, VlakRen *vlr, floa
 		nor[0]= imat[0][0]*xn+imat[0][1]*yn+imat[0][2]*zn;
 		nor[1]= imat[1][0]*xn+imat[1][1]*yn+imat[1][2]*zn;
 		nor[2]= imat[2][0]*xn+imat[2][1]*yn+imat[2][2]*zn;
+		Normalize(nor);
 	}
 	else
 		VECCOPY(nor, vlr->n);
@@ -1275,34 +1276,9 @@ ObjectInstanceRen *RE_addRenderInstance(Render *re, ObjectRen *obr, Object *ob,
 	return obi;
 }
 
-void find_dupli_objectren(Render *re, ObjectInstanceRen *obi, ObjectInstanceRen *lastobi)
-{
-	ObjectRen *obr;
-
-	/* see if last object did the same lookup, so we can just reuse result */
-	if(lastobi && obi->ob == lastobi->ob && obi->par == lastobi->par && obi->psysindex == lastobi->psysindex) {
-		obi->obr= lastobi->obr;
-		return;
-	}
-
-	/* dupli objects are created after object instances, so we look through
-	 * object list to find it */
-	obr= re->objecttable.first;
-	while(obr && (obr->ob != obi->ob || obr->par != obi->par || obr->psysindex != obi->psysindex))
-		obr= obr->next;
-
-	if(!obr) {
-		obr= re->objecttable.first;
-		while(obr && (obr->ob != obi->ob || obr->psysindex != obi->psysindex) && obr->par == NULL)
-			obr= obr->next;
-	}
-
-	obi->obr= obr;
-}
-
 void RE_makeRenderInstances(Render *re)
 {
-	ObjectInstanceRen *obi, *oldobi, *lastobi= NULL;
+	ObjectInstanceRen *obi, *oldobi;
 	ListBase newlist;
 	int tot;
 
@@ -1316,11 +1292,6 @@ void RE_makeRenderInstances(Render *re)
 	for(oldobi=re->instancetable.first; oldobi; oldobi=oldobi->next) {
 		*obi= *oldobi;
 
-		if(!obi->obr) {
-			find_dupli_objectren(re, obi, lastobi);
-			lastobi= obi;
-		}
-
 		if(obi->obr) {
 			obi->prev= obi->next= NULL;
 			BLI_addtail(&newlist, obi);