Sculpt: external file storage for multires

* This is experimental, the file format may change still!
* Helps reduce memory usage, keeps .blend files smaller, and makes
  saving quicker when not editing multires.
* This is implemented at the customdata level, currently only the
  multires displacements can be stored externally.

ToDo

* Better integration with object duplication/removal/..
* Memory is not yet freed when exiting sculpt mode.
* Loading only lower levels is not supported yet.

release/scripts/ui/properties_data_modifier.py

@@ -302,9 +302,9 @@ class DATA_PT_modifiers(DataButtonsPanel):
         col = split.column()
         col.itemO("object.multires_subdivide", text="Subdivide")
         col.itemO("object.multires_higher_levels_delete", text="Delete Higher")
-        # row = col.row()
-        # row.enabled = md.total_levels > 0
-        # row.itemR(md, "external")
+        row = col.row()
+        row.enabled = md.total_levels > 0
+        row.itemR(md, "external")
 
     def PARTICLE_INSTANCE(self, layout, ob, md):
         layout.itemR(md, "object")

source/blender/blenkernel/BKE_btex.h

@@ -0,0 +1,64 @@
+/*
+ * $Id$
+ *
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#ifndef BKE_BTEX_H
+#define BKE_BTEX_H
+
+#define BTEX_TYPE_IMAGE	0
+#define BTEX_TYPE_MESH	1
+
+#define BTEX_LAYER_NAME_MAX	64
+
+typedef struct BTex BTex;
+typedef struct BTexLayer BTexLayer;
+
+/* Create/Free */
+
+BTex *btex_create(int type);
+void btex_free(BTex *btex);
+
+/* File read/write/remove */
+
+int btex_read_open(BTex *btex, char *filename);
+int btex_read_layer(BTex *btex, BTexLayer *blay);
+int btex_read_data(BTex *btex, int size, void *data);
+void btex_read_close(BTex *btex);
+
+int btex_write_open(BTex *btex, char *filename);
+int btex_write_layer(BTex *btex, BTexLayer *blay);
+int btex_write_data(BTex *btex, int size, void *data);
+void btex_write_close(BTex *btex);
+
+void btex_remove(char *filename);
+
+/* Layers */
+
+BTexLayer *btex_layer_find(BTex *btex, int type, char *name);
+BTexLayer *btex_layer_add(BTex *btex, int type, char *name);
+void btex_layer_remove(BTex *btex, BTexLayer *blay);
+
+/* Mesh */
+
+void btex_mesh_set_grids(BTex *btex, int totgrid, int gridsize, int datasize);
+
+#endif /* BKE_BTEX_H */
+

source/blender/blenkernel/BKE_customdata.h

@@ -278,4 +278,20 @@ int CustomData_verify_versions(struct CustomData *data, int index);
 void CustomData_to_bmeshpoly(struct CustomData *fdata, struct CustomData *pdata, struct CustomData *ldata);
 void CustomData_from_bmeshpoly(struct CustomData *fdata, struct CustomData *pdata, struct CustomData *ldata, int total);
 void CustomData_bmesh_init_pool(struct CustomData *data, int allocsize);
+
+/* External file storage */
+
+void CustomData_external_add(struct CustomData *data,
+	int type, const char *name, int totelem);
+void CustomData_external_remove(struct CustomData *data,
+	int type, int totelem);
+void CustomData_external_remove_object(struct CustomData *data);
+int CustomData_external_test(struct CustomData *data, int type);
+
+void CustomData_external_write(struct CustomData *data,
+	CustomDataMask mask, int totelem, int free);
+void CustomData_external_read(struct CustomData *data,
+	CustomDataMask mask, int totelem);
+
 #endif
+

source/blender/blenkernel/intern/btex.c

@@ -0,0 +1,482 @@
+/*
+ * $Id$
+ *
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "MEM_guardedalloc.h"
+
+#include "BLI_fileops.h"
+#include "BLI_string.h"
+
+#include "BKE_btex.h"
+#include "BKE_global.h"
+#include "BKE_utildefines.h"
+
+/************************* File Format Definitions ***************************/
+
+#define BTEX_ENDIAN_LITTLE	0
+#define BTEX_ENDIAN_BIG		1
+
+#define BTEX_DATA_FLOAT	0
+
+typedef struct BTexHeader {
+	char ID[4];					/* "BTEX" */
+	char endian;				/* little, big */
+	char version;				/* non-compatible versions */
+	char subversion;			/* compatible sub versions */
+	char pad;					/* padding */
+
+	int structbytes;			/* size of this struct in bytes */
+	int type;					/* image, mesh */
+	int totlayer;				/* number of layers in the file */
+} BTexHeader;
+
+typedef struct BTexImageHeader {
+	int structbytes;			/* size of this struct in bytes */
+	int width;					/* image width */
+	int height;					/* image height */
+	int tile_size;				/* tile size (required power of 2) */
+} BTexImageHeader;
+
+typedef struct BTexMeshHeader {
+	int structbytes;			/* size of this struct in bytes */
+	int totgrid;				/* number of grids */
+	int gridsize;				/* width of grids */
+} BTexMeshHeader;
+
+struct BTexLayer {
+	int structbytes;				/* size of this struct in bytes */
+	int datatype;					/* only float for now */
+	int datasize;					/* size of data in layer */
+	int type;						/* layer type */
+	char name[BTEX_LAYER_NAME_MAX];	/* layer name */
+};
+
+/**************************** Other Definitions ******************************/
+
+#define BTEX_VERSION		0
+#define BTEX_SUBVERSION		0
+#define BTEX_TILE_SIZE		64
+
+struct BTex {
+	int type;
+
+	BTexHeader header;
+	union {
+		BTexImageHeader image;
+		BTexMeshHeader mesh;
+	} btype;
+
+	BTexLayer *layer;
+	int totlayer;
+
+	FILE *readf;
+	FILE *writef;
+	int switchendian;
+	size_t dataoffset;
+};
+
+/********************************* Create/Free *******************************/
+
+static int btex_endian(void)
+{
+	if(ENDIAN_ORDER == L_ENDIAN)
+		return BTEX_ENDIAN_LITTLE;
+	else
+		return BTEX_ENDIAN_BIG;
+}
+
+/*static int btex_data_type_size(int datatype)
+{
+	if(datatype == BTEX_DATA_FLOAT)
+		return sizeof(float);
+	
+	return 0;
+}*/
+
+BTex *btex_create(int type)
+{
+	BTex *btex= MEM_callocN(sizeof(BTex), "BTex");
+
+	btex->type= type;
+
+	return btex;
+}
+
+void btex_free(BTex *btex)
+{
+	btex_read_close(btex);
+	btex_write_close(btex);
+
+	if(btex->layer)
+		MEM_freeN(btex->layer);
+
+	MEM_freeN(btex);
+}
+
+/********************************* Read/Write ********************************/
+
+static int btex_read_header(BTex *btex)
+{
+	BTexHeader *header;
+	BTexImageHeader *image;
+	BTexMeshHeader *mesh;
+	BTexLayer *layer;
+	FILE *f= btex->readf;
+	size_t offset = 0;
+	int a;
+
+	header= &btex->header;
+
+	if(!fread(header, sizeof(BTexHeader), 1, btex->readf))
+		return 0;
+	
+	if(memcmp(header->ID, "BTEX", sizeof(header->ID)) != 0)
+		return 0;
+	if(header->version > BTEX_VERSION)
+		return 0;
+
+	btex->switchendian= header->endian != btex_endian();
+	header->endian= btex_endian();
+
+	if(btex->switchendian) {
+		SWITCH_INT(header->type);
+		SWITCH_INT(header->totlayer);
+		SWITCH_INT(header->structbytes);
+	}
+
+	if(!ELEM(header->type, BTEX_TYPE_IMAGE, BTEX_TYPE_MESH))
+		return 0;
+
+	offset += header->structbytes;
+	header->structbytes= sizeof(BTexHeader);
+
+	if(fseek(f, offset, SEEK_SET) != 0)
+		return 0;
+	
+	if(header->type == BTEX_TYPE_IMAGE) {
+		image= &btex->btype.image;
+		if(!fread(image, sizeof(BTexImageHeader), 1, f))
+			return 0;
+
+		if(btex->switchendian) {
+			SWITCH_INT(image->width);
+			SWITCH_INT(image->height);
+			SWITCH_INT(image->tile_size);
+			SWITCH_INT(image->structbytes);
+		}
+
+		offset += image->structbytes;
+		image->structbytes= sizeof(BTexImageHeader);
+	}
+	else if(header->type == BTEX_TYPE_MESH) {
+		mesh= &btex->btype.mesh;
+		if(!fread(mesh, sizeof(BTexMeshHeader), 1, f))
+			return 0;
+
+		if(btex->switchendian) {
+			SWITCH_INT(mesh->totgrid);
+			SWITCH_INT(mesh->gridsize);
+			SWITCH_INT(mesh->structbytes);
+		}
+
+		offset += mesh->structbytes;
+		mesh->structbytes= sizeof(BTexMeshHeader);
+	}
+
+	if(fseek(f, offset, SEEK_SET) != 0)
+		return 0;
+
+	btex->layer= MEM_callocN(sizeof(BTexLayer)*header->totlayer, "BTexLayer");
+	btex->totlayer= header->totlayer;
+
+	for(a=0; a<header->totlayer; a++) {
+		layer= &btex->layer[a];
+
+		if(!fread(layer, sizeof(BTexLayer), 1, f))
+			return 0;
+
+		if(btex->switchendian) {
+			SWITCH_INT(layer->type);
+			SWITCH_INT(layer->datatype);
+			SWITCH_INT(layer->datasize);
+			SWITCH_INT(layer->structbytes);
+		}
+
+		if(layer->datatype != BTEX_DATA_FLOAT)
+			return 0;
+
+		offset += layer->structbytes;
+		layer->structbytes= sizeof(BTexLayer);
+
+		if(fseek(f, offset, SEEK_SET) != 0)
+			return 0;
+	}
+
+	btex->dataoffset= offset;
+
+	return 1;
+}
+
+static int btex_write_header(BTex *btex)
+{
+	BTexHeader *header;
+	BTexImageHeader *image;
+	BTexMeshHeader *mesh;
+	BTexLayer *layer;
+	FILE *f= btex->writef;
+	int a;
+
+	header= &btex->header;
+
+	if(!fwrite(header, sizeof(BTexHeader), 1, f))
+		return 0;
+	
+	if(header->type == BTEX_TYPE_IMAGE) {
+		image= &btex->btype.image;
+		if(!fwrite(image, sizeof(BTexImageHeader), 1, f))
+			return 0;
+	}
+	else if(header->type == BTEX_TYPE_MESH) {
+		mesh= &btex->btype.mesh;
+		if(!fwrite(mesh, sizeof(BTexMeshHeader), 1, f))
+			return 0;
+	}
+
+	for(a=0; a<header->totlayer; a++) {
+		layer= &btex->layer[a];
+
+		if(!fwrite(layer, sizeof(BTexLayer), 1, f))
+			return 0;
+	}
+
+	return 1;
+}
+
+int btex_read_open(BTex *btex, char *filename)
+{
+	FILE *f;
+
+	f= fopen(filename, "rb");
+	if(!f)
+		return 0;
+	
+	btex->readf= f;
+
+	if(!btex_read_header(btex)) {
+		btex_read_close(btex);
+		return 0;
+	}
+
+	if(btex->header.type != btex->type) {
+		btex_read_close(btex);
+		return 0;
+	}
+
+	return 1;
+}
+
+int btex_read_layer(BTex *btex, BTexLayer *blay)
+{
+	size_t offset;
+	int a;
+
+	/* seek to right location in file */
+	offset= btex->dataoffset;
+	for(a=0; a<btex->totlayer; a++) {
+		if(&btex->layer[a] == blay)
+			break;
+		else
+			offset += btex->layer[a].datasize;
+	}
+
+	return (fseek(btex->readf, offset, SEEK_SET) == 0);
+}
+
+int btex_read_data(BTex *btex, int size, void *data)
+{
+	float *fdata;
+	int a;
+	
+	/* read data */
+	if(!fread(data, size, 1, btex->readf))
+		return 0;
+
+	/* switch endian if necessary */
+	if(btex->switchendian) {
+		fdata= data;
+
+		for(a=0; a<size/sizeof(float); a++)
+			SWITCH_INT(fdata[a])
+	}
+
+	return 1;
+}
+
+void btex_read_close(BTex *btex)
+{
+	if(btex->readf) {
+		fclose(btex->readf);
+		btex->readf= NULL;
+	}
+}
+
+int btex_write_open(BTex *btex, char *filename)
+{
+	BTexHeader *header;
+	BTexImageHeader *image;
+	BTexMeshHeader *mesh;
+	FILE *f;
+
+	f= fopen(filename, "wb");
+	if(!f)
+		return 0;
+	
+	btex->writef= f;
+
+	/* fill header */
+	header= &btex->header;
+	strcpy(header->ID, "BTEX");
+	header->endian= btex_endian();
+	header->version= BTEX_VERSION;
+	header->subversion= BTEX_SUBVERSION;
+
+	header->structbytes= sizeof(BTexHeader);
+	header->type= btex->type;
+	header->totlayer= btex->totlayer;
+
+	if(btex->type == BTEX_TYPE_IMAGE) {
+		/* fill image header */
+		image= &btex->btype.image;
+		image->structbytes= sizeof(BTexImageHeader);
+		image->tile_size= BTEX_TILE_SIZE;
+	}
+	else if(btex->type == BTEX_TYPE_MESH) {
+		/* fill mesh header */
+		mesh= &btex->btype.mesh;
+		mesh->structbytes= sizeof(BTexMeshHeader);
+	}
+
+	btex_write_header(btex);
+
+	return 1;
+}
+
+int btex_write_layer(BTex *btex, BTexLayer *blay)
+{
+	return 1;
+}
+
+int btex_write_data(BTex *btex, int size, void *data)
+{
+	/* write data */
+	if(!fwrite(data, size, 1, btex->writef))
+		return 0;
+
+	return 1;
+}
+
+void btex_write_close(BTex *btex)
+{
+	if(btex->writef) {
+		fclose(btex->writef);
+		btex->writef= NULL;
+	}
+}
+
+void btex_remove(char *filename)
+{
+	BLI_delete(filename, 0, 0);
+}
+
+/********************************** Layers ***********************************/
+
+BTexLayer *btex_layer_find(BTex *btex, int type, char *name)
+{
+	BTexLayer *layer;
+	int a;
+
+	for(a=0; a<btex->totlayer; a++) {
+		layer= &btex->layer[a];
+
+		if(layer->type == type && strcmp(layer->name, name) == 0)
+			return layer;
+	}
+	
+	return NULL;
+}
+
+BTexLayer *btex_layer_add(BTex *btex, int type, char *name)
+{
+	BTexLayer *newlayer, *layer;
+
+	/* expand array */
+	newlayer= MEM_callocN(sizeof(BTexLayer)*(btex->totlayer+1), "BTexLayer");
+	memcpy(newlayer, btex->layer, sizeof(BTexLayer)*btex->totlayer);
+	btex->layer= newlayer;
+
+	btex->totlayer++;
+
+	/* fill in new layer */
+	layer= &btex->layer[btex->totlayer-1];
+	layer->structbytes= sizeof(BTexLayer);
+	layer->datatype= BTEX_DATA_FLOAT;
+	layer->type= type;
+	BLI_strncpy(layer->name, name, BTEX_LAYER_NAME_MAX);
+
+	return layer;
+}
+
+void btex_layer_remove(BTex *btex, BTexLayer *layer)
+{
+	BTexLayer *newlayer;
+	int index= layer - btex->layer;
+
+	/* expand array */
+	newlayer= MEM_callocN(sizeof(BTexLayer)*(btex->totlayer-1), "BTexLayer");
+	if(index > 0)
+		memcpy(newlayer, btex->layer, sizeof(BTexLayer)*index);
+	if(index+1 < btex->totlayer)
+		memcpy(newlayer+index, btex->layer+index+1, sizeof(BTexLayer)*(btex->totlayer-(index+1)));
+	btex->layer= newlayer;
+
+	btex->totlayer--;
+}
+
+/********************************* Mesh **************************************/
+
+void btex_mesh_set_grids(BTex *btex, int totgrid, int gridsize, int datasize)
+{
+	BTexLayer *layer;
+	int a;
+
+	btex->btype.mesh.totgrid= totgrid;
+	btex->btype.mesh.gridsize= gridsize;
+
+	for(a=0; a<btex->totlayer; a++) {
+		layer= &btex->layer[a];
+		layer->datasize= datasize;
+	}
+}
+

source/blender/blenkernel/intern/customdata.c

@@ -32,21 +32,26 @@
 *
 */ 
 
-#include "BKE_customdata.h"
-#include "BKE_utildefines.h" // CLAMP
-#include "BLI_math.h"
-#include "BLI_blenlib.h"
-#include "BLI_linklist.h"
-#include "BLI_mempool.h"
+#include <math.h>
+#include <string.h>
+
+#include "MEM_guardedalloc.h"
 
 #include "DNA_customdata_types.h"
 #include "DNA_listBase.h"
 #include "DNA_meshdata_types.h"
+#include "DNA_ID.h"
 
-#include "MEM_guardedalloc.h"
+#include "BLI_blenlib.h"
+#include "BLI_linklist.h"
+#include "BLI_math.h"
+#include "BLI_mempool.h"
+#include "BLI_string.h"
 
-#include <math.h>
-#include <string.h>
+#include "BKE_btex.h"
+#include "BKE_customdata.h"
+#include "BKE_global.h"
+#include "BKE_utildefines.h"
 
 /* number of layers to add when growing a CustomData object */
 #define CUSTOMDATA_GROW 5
@@ -89,6 +94,12 @@ typedef struct LayerTypeInfo {
     /* a function to set a layer's data to default values. if NULL, the
 	   default is assumed to be all zeros */
 	void (*set_default)(void *data, int count);
+
+    /* a function to read data from a btex file */
+	int (*read)(BTex *btex, void *data, int count);
+
+    /* a function to write data to a btex file */
+	int (*write)(BTex *btex, void *data, int count);
 } LayerTypeInfo;
 
 static void layerCopy_mdeformvert(const void *source, void *dest,
@@ -538,6 +549,39 @@ static void layerFree_mdisps(void *data, int count, int size)
 	}
 }
 
+static int layerRead_mdisps(BTex *btex, void *data, int count)
+{
+	MDisps *d = data;
+	int i;
+
+	for(i = 0; i < count; ++i) {
+		if(!d[i].disps)
+			d[i].disps = MEM_callocN(sizeof(float)*3*d[i].totdisp, "mdisps read");
+
+		if(!btex_read_data(btex, d[i].totdisp*3*sizeof(float), d[i].disps)) {
+			printf("failed to read %d/%d %d\n", i, count, d[i].totdisp);
+			return 0;
+		}
+	}
+
+	return 1;
+}
+
+static int layerWrite_mdisps(BTex *btex, void *data, int count)
+{
+	MDisps *d = data;
+	int i;
+
+	for(i = 0; i < count; ++i) {
+		if(!btex_write_data(btex, d[i].totdisp*3*sizeof(float), d[i].disps)) {
+			printf("failed to write %d/%d %d\n", i, count, d[i].totdisp);
+			return 0;
+		}
+	}
+
+	return 1;
+}
+
 /* --------- */
 
 static void layerDefault_mloopcol(void *data, int count)
@@ -731,7 +775,7 @@ const LayerTypeInfo LAYERTYPEINFO[CD_NUMTYPES] = {
 	{sizeof(MLoopCol), "MLoopCol", 1, "Col", NULL, NULL, layerInterp_mloopcol, NULL, layerDefault_mloopcol},
 	{sizeof(float)*3*4, "", 0, NULL, NULL, NULL, NULL, NULL, NULL},
 	{sizeof(MDisps), "MDisps", 1, NULL, layerCopy_mdisps,
-	 layerFree_mdisps, layerInterp_mdisps, layerSwap_mdisps, NULL},
+	 layerFree_mdisps, layerInterp_mdisps, layerSwap_mdisps, NULL, layerRead_mdisps, layerWrite_mdisps},
 	{sizeof(MCol)*4, "MCol", 4, "WeightCol", NULL, NULL, layerInterp_mcol,
 	 layerSwap_mcol, layerDefault_mcol},
 	 {sizeof(MCol)*4, "MCol", 4, "IDCol", NULL, NULL, layerInterp_mcol,
@@ -793,6 +837,8 @@ void CustomData_merge(const struct CustomData *source, struct CustomData *dest,
 	CustomDataLayer *layer, *newlayer;
 	int i, type, number = 0, lasttype = -1, lastactive = 0, lastrender = 0, lastclone = 0, lastmask = 0;
 
+	CustomData_external_read(dest, mask, totelem);
+
 	for(i = 0; i < source->totlayer; ++i) {
 		layer = &source->layers[i];
 		typeInfo = layerType_getInfo(layer->type);
@@ -853,6 +899,14 @@ static void customData_free_layer__internal(CustomDataLayer *layer, int totelem)
 	}
 }
 
+static void CustomData_external_free(CustomData *data)
+{
+	if(data->external) {
+		MEM_freeN(data->external);
+		data->external= NULL;
+	}
+}
+
 void CustomData_free(CustomData *data, int totelem)
 {
 	int i;
@@ -863,6 +917,8 @@ void CustomData_free(CustomData *data, int totelem)
 	if(data->layers)
 		MEM_freeN(data->layers);
 	
+	CustomData_external_free(data);
+	
 	memset(data, 0, sizeof(*data));
 }
 
@@ -2238,3 +2294,232 @@ int CustomData_verify_versions(struct CustomData *data, int index)
 	return keeplayer;
 }
 
+/****************************** External Files *******************************/
+
+static void customdata_external_filename(char filename[FILE_MAX], CustomDataExternal *external)
+{
+	BLI_strncpy(filename, external->filename, FILE_MAX);
+	BLI_convertstringcode(filename, G.sce);
+}
+
+void CustomData_external_read(CustomData *data, CustomDataMask mask, int totelem)
+{
+	CustomDataExternal *external= data->external;
+	CustomDataLayer *layer;
+	BTex *btex;
+	BTexLayer *blay;
+	char filename[FILE_MAX];
+	const LayerTypeInfo *typeInfo;
+	int i, update = 0;
+
+	if(!external)
+		return;
+	
+	for(i=0; i<data->totlayer; i++) {
+		layer = &data->layers[i];
+		typeInfo = layerType_getInfo(layer->type);
+
+		if(!(mask & (1<<layer->type)));
+		else if(layer->flag & CD_FLAG_IN_MEMORY);
+		else if((layer->flag & CD_FLAG_EXTERNAL) && typeInfo->read)
+			update= 1;
+	}
+
+	if(!update)
+		return;
+
+	customdata_external_filename(filename, external);
+
+	btex= btex_create(BTEX_TYPE_MESH);
+	if(!btex_read_open(btex, filename))
+		return;
+
+	for(i=0; i<data->totlayer; i++) {
+		layer = &data->layers[i];
+		typeInfo = layerType_getInfo(layer->type);
+
+		if(!(mask & (1<<layer->type)));
+		else if(layer->flag & CD_FLAG_IN_MEMORY);
+		else if((layer->flag & CD_FLAG_EXTERNAL) && typeInfo->read) {
+			blay= btex_layer_find(btex, layer->type, layer->name);
+
+			if(blay) {
+				if(btex_read_layer(btex, blay)) {
+					if(typeInfo->read(btex, layer->data, totelem));
+					else break;
+					layer->flag |= CD_FLAG_IN_MEMORY;
+				}
+				else
+					break;
+			}
+		}
+	}
+
+	btex_read_close(btex);
+	btex_free(btex);
+}
+
+void CustomData_external_write(CustomData *data, CustomDataMask mask, int totelem, int free)
+{
+	CustomDataExternal *external= data->external;
+	CustomDataLayer *layer;
+	BTex *btex;
+	BTexLayer *blay;
+	const LayerTypeInfo *typeInfo;
+	int i, update = 0;
+	char filename[FILE_MAX];
+
+	if(!external)
+		return;
+
+	for(i=0; i<data->totlayer; i++) {
+		layer = &data->layers[i];
+		typeInfo = layerType_getInfo(layer->type);
+
+		if(!(mask & (1<<layer->type)));
+		else if((layer->flag & CD_FLAG_EXTERNAL) && typeInfo->write)
+			update= 1;
+	}
+
+	if(!update)
+		return;
+
+	CustomData_external_read(data, mask, totelem);
+
+	btex= btex_create(BTEX_TYPE_MESH);
+
+	for(i=0; i<data->totlayer; i++) {
+		layer = &data->layers[i];
+		typeInfo = layerType_getInfo(layer->type);
+
+		if((layer->flag & CD_FLAG_EXTERNAL) && typeInfo->write)
+			btex_layer_add(btex, layer->type, layer->name);
+	}
+
+	customdata_external_filename(filename, external);
+	if(!btex_write_open(btex, filename))
+		return;
+
+	for(i=0; i<data->totlayer; i++) {
+		layer = &data->layers[i];
+		typeInfo = layerType_getInfo(layer->type);
+
+		if((layer->flag & CD_FLAG_EXTERNAL) && typeInfo->write) {
+			blay= btex_layer_find(btex, layer->type, layer->name);
+
+			if(btex_write_layer(btex, blay)) {
+				if(typeInfo->write(btex, layer->data, totelem));
+				else break;
+			}
+			else
+				break;
+		}
+	}
+
+	if(i != data->totlayer) {
+		btex_free(btex);
+		return;
+	}
+
+	for(i=0; i<data->totlayer; i++) {
+		layer = &data->layers[i];
+		typeInfo = layerType_getInfo(layer->type);
+
+		if((layer->flag & CD_FLAG_EXTERNAL) && typeInfo->write) {
+			if(free) {
+				if(typeInfo->free)
+					typeInfo->free(layer->data, totelem, typeInfo->size);
+				layer->flag &= ~CD_FLAG_IN_MEMORY;
+			}
+		}
+	}
+
+	btex_write_close(btex);
+	btex_free(btex);
+}
+
+void CustomData_external_add(CustomData *data, int type, const char *name, int totelem)
+{
+	CustomDataExternal *external= data->external;
+	CustomDataLayer *layer;
+	int layer_index;
+
+	layer_index = CustomData_get_active_layer_index(data, type);
+	if(layer_index < 0) return;
+
+	layer = &data->layers[layer_index];
+
+	if(layer->flag & CD_FLAG_EXTERNAL)
+		return;
+
+	if(!external) {
+		char hex[MAX_ID_NAME*2];
+
+		external= MEM_callocN(sizeof(CustomDataExternal), "CustomDataExternal");
+		BLI_strhex(hex, sizeof(hex), name);
+		BLI_snprintf(external->filename, sizeof(external->filename), "//%s_mesh.btex", hex);
+		data->external= external;
+	}
+
+	layer->flag |= CD_FLAG_EXTERNAL|CD_FLAG_IN_MEMORY;
+}
+
+void CustomData_external_remove(CustomData *data, int type, int totelem)
+{
+	CustomDataExternal *external= data->external;
+	CustomDataLayer *layer;
+	char filename[FILE_MAX];
+	int layer_index, i, remove_file;
+
+	layer_index = CustomData_get_active_layer_index(data, type);
+	if(layer_index < 0) return;
+
+	layer = &data->layers[layer_index];
+
+	if(!external)
+		return;
+
+	if(layer->flag & CD_FLAG_EXTERNAL) {
+		if(!(layer->flag & CD_FLAG_IN_MEMORY))
+			CustomData_external_read(data, (1<<layer->type), totelem);
+
+		layer->flag &= ~CD_FLAG_EXTERNAL;
+
+		remove_file= 1;
+		for(i=0; i<data->totlayer; i++)
+			if(data->layers[i].flag & CD_FLAG_EXTERNAL)
+				remove_file= 0;
+
+		if(remove_file) {
+			customdata_external_filename(filename, external);
+			btex_remove(filename);
+			CustomData_external_free(data);
+		}
+	}
+}
+
+int CustomData_external_test(CustomData *data, int type)
+{
+	CustomDataLayer *layer;
+	int layer_index;
+
+	layer_index = CustomData_get_active_layer_index(data, type);
+	if(layer_index < 0) return 0;
+
+	layer = &data->layers[layer_index];
+	return (layer->flag & CD_FLAG_EXTERNAL);
+}
+
+void CustomData_external_remove_object(CustomData *data)
+{
+	CustomDataExternal *external= data->external;
+	char filename[FILE_MAX];
+
+	if(!external)
+		return;
+
+	customdata_external_filename(filename, external);
+	btex_remove(filename);
+	CustomData_external_free(data);
+}
+

source/blender/blenkernel/intern/multires.c

@@ -249,7 +249,7 @@ void multiresModifier_del_levels(struct MultiresModifierData *mmd, struct Object
 	int levels = mmd->totlvl - lvl;
 	MDisps *mdisps;
 	
-	// XXX CustomData_external_read(&me->fdata, CD_MASK_MDISPS, me->totface);
+	CustomData_external_read(&me->fdata, CD_MASK_MDISPS, me->totface);
 	mdisps= CustomData_get_layer(&me->fdata, CD_MDISPS);
 
 	multires_force_update(ob);
@@ -595,7 +595,7 @@ static void multiresModifier_update(DerivedMesh *dm)
 	ob = ccgdm->multires.ob;
 	me = ccgdm->multires.ob->data;
 	mmd = ccgdm->multires.mmd;
-	// XXX CustomData_external_read(&me->fdata, CD_MASK_MDISPS, me->totface);
+	CustomData_external_read(&me->fdata, CD_MASK_MDISPS, me->totface);
 	mdisps = CustomData_get_layer(&me->fdata, CD_MDISPS);
 
 	if(mdisps) {
@@ -692,7 +692,7 @@ void multires_force_update(Object *ob)
 struct DerivedMesh *multires_dm_create_from_derived(MultiresModifierData *mmd, int local_mmd, DerivedMesh *dm, Object *ob,
 						    int useRenderParams, int isFinalCalc)
 {
-	//Mesh *me= ob->data;
+	Mesh *me= ob->data;
 	DerivedMesh *result;
 	CCGDerivedMesh *ccgdm;
 	DMGridData **gridData, **subGridData;
@@ -727,7 +727,7 @@ struct DerivedMesh *multires_dm_create_from_derived(MultiresModifierData *mmd, i
 		memcpy(subGridData[i], gridData[i], sizeof(DMGridData)*gridSize*gridSize);
 	}
 
-	// XXX CustomData_external_read(&me->fdata, CD_MASK_MDISPS, me->totface);
+	CustomData_external_read(&me->fdata, CD_MASK_MDISPS, me->totface);
 	multiresModifier_disp_run(result, ob->data, 0, 0, subGridData, mmd->totlvl);
 
 	for(i = 0; i < numGrids; i++)

source/blender/blenkernel/intern/pointcache.c

@@ -1075,12 +1075,11 @@ static int BKE_ptcache_id_filename(PTCacheID *pid, char *filename, int cfra, sho
 	}
 	if(strcmp(pid->cache->name, "")==0 && (pid->cache->flag & PTCACHE_EXTERNAL)==0) {
 		idname = (pid->ob->id.name+2);
+
 		/* convert chars to hex so they are always a valid filename */
-		while('\0' != *idname) {
-			snprintf(newname, MAX_PTCACHE_FILE, "%02X", (char)(*idname++));
-			newname+=2;
-			len += 2;
-		}
+		BLI_strhex(newname, MAX_PTCACHE_FILE - len, idname);
+		len += strlen(newname);
+		newname = filename + len;
 	}
 	else {
 		int temp = (int)strlen(pid->cache->name); 

source/blender/blenlib/BLI_string.h

@@ -123,6 +123,9 @@ int BLI_natstrcmp(const char *s1, const char *s2);
 
 void BLI_timestr(double _time, char *str); /* time var is global */
 
+	/* Convert to hex string valid for file writing (2x length of original) */
+void BLI_strhex(char *hex, int maxlen, const char *str);
+
 #ifdef __cplusplus
 }
 #endif

source/blender/blenlib/intern/string.c

@@ -329,3 +329,18 @@ void BLI_timestr(double _time, char *str)
 	
 	str[11]=0;
 }
+
+void BLI_strhex(char *hex, int maxlen, const char *str)
+{
+	int len = 0;
+
+	while('\0' != *str && len+3 < maxlen) {
+		snprintf(hex, maxlen, "%02X", *str++);
+		hex += 2;
+		len += 2;
+	}
+
+	if(maxlen)
+		hex[0]= '\0';
+}
+

source/blender/blenloader/intern/readfile.c

@@ -3311,14 +3311,14 @@ static void direct_link_dverts(FileData *fd, int count, MDeformVert *mdverts)
 	}
 }
 
-static void direct_link_mdisps(FileData *fd, int count, MDisps *mdisps)
+static void direct_link_mdisps(FileData *fd, int count, MDisps *mdisps, int external)
 {
 	if(mdisps) {
 		int i;
 
 		for(i = 0; i < count; ++i) {
 			mdisps[i].disps = newdataadr(fd, mdisps[i].disps);
-			if(!mdisps[i].disps)
+			if(!external && !mdisps[i].disps)
 				mdisps[i].totdisp = 0;
 		}
 	}       
@@ -3329,14 +3329,18 @@ static void direct_link_customdata(FileData *fd, CustomData *data, int count)
 	int i = 0;
 
 	data->layers= newdataadr(fd, data->layers);
+	data->external= newdataadr(fd, data->external);
 
 	while (i < data->totlayer) {
 		CustomDataLayer *layer = &data->layers[i];
 
+		if(layer->flag & CD_FLAG_EXTERNAL)
+			layer->flag &= ~CD_FLAG_IN_MEMORY;
+
 		if (CustomData_verify_versions(data, i)) {
 			layer->data = newdataadr(fd, layer->data);
 			if(layer->type == CD_MDISPS)
-				direct_link_mdisps(fd, count, layer->data);
+				direct_link_mdisps(fd, count, layer->data, layer->flag & CD_FLAG_EXTERNAL);
 			i++;
 		}
 	}

source/blender/blenloader/intern/writefile.c

@@ -1434,23 +1434,29 @@ static void write_dverts(WriteData *wd, int count, MDeformVert *dvlist)
 	}
 }
 
-static void write_mdisps(WriteData *wd, int count, MDisps *mdlist)
+static void write_mdisps(WriteData *wd, int count, MDisps *mdlist, int external)
 {
 	if(mdlist) {
 		int i;
 		
 		writestruct(wd, DATA, "MDisps", count, mdlist);
+		if(!external) {
 			for(i = 0; i < count; ++i) {
 				if(mdlist[i].disps)
 					writedata(wd, DATA, sizeof(float)*3*mdlist[i].totdisp, mdlist[i].disps);
 			}
 		}
+	}
 }
 
 static void write_customdata(WriteData *wd, int count, CustomData *data, int partial_type, int partial_count)
 {
 	int i;
 
+	/* write external customdata */
+	if(data->external && !wd->current)
+		CustomData_external_write(data, CD_MASK_MESH, count, 0);
+
 	writestruct(wd, DATA, "CustomDataLayer", data->maxlayer, data->layers);
 
 	for (i=0; i<data->totlayer; i++) {
@@ -1463,7 +1469,7 @@ static void write_customdata(WriteData *wd, int count, CustomData *data, int par
 			write_dverts(wd, count, layer->data);
 		}
 		else if (layer->type == CD_MDISPS) {
-			write_mdisps(wd, count, layer->data);
+			write_mdisps(wd, count, layer->data, layer->flag & CD_FLAG_EXTERNAL);
 		}
 		else {
 			CustomData_file_write_info(layer->type, &structname, &structnum);
@@ -1480,6 +1486,9 @@ static void write_customdata(WriteData *wd, int count, CustomData *data, int par
 				printf("error: this CustomDataLayer must not be written to file\n");
 		}
 	}
+
+	if(data->external)
+		writestruct(wd, DATA, "CustomDataExternal", 1, data->external);
 }
 
 static void write_meshs(WriteData *wd, ListBase *idbase)

source/blender/editors/include/ED_sculpt.h

@@ -39,7 +39,7 @@ struct wmWindowManager;
 void ED_operatortypes_sculpt(void);
 void sculpt_get_redraw_planes(float planes[4][4], struct ARegion *ar,
  			      struct RegionView3D *rv3d, struct Object *ob);
-
+void ED_sculpt_force_update(struct bContext *C);
 
 /* paint_ops.c */
 void ED_operatortypes_paint(void);

source/blender/makesdna/DNA_customdata_types.h

@@ -44,6 +44,10 @@ typedef struct CustomDataLayer {
 	void *data;     /* layer data */
 } CustomDataLayer;
 
+typedef struct CustomDataExternal {
+	char filename[240]; /* FILE_MAX */
+} CustomDataExternal;
+
 /* structure which stores custom element data associated with mesh elements
  * (vertices, edges or faces). The custom data is organised into a series of
  * layers, each with a data type (e.g. MTFace, MDeformVert, etc.). */
@@ -51,7 +55,8 @@ typedef struct CustomData {
 	CustomDataLayer *layers;      /* CustomDataLayers, ordered by type */
 	int totlayer, maxlayer;       /* number of layers, size of layers array */
 	int totsize, pad;             /* in editmode, total size of all data layers */
-	void *pool;		  /* for Bmesh: Memory pool for allocation of blocks*/
+	void *pool;                   /* Bmesh: Memory pool for allocation of blocks */
+	CustomDataExternal *external; /* external file storing customdata layers */
 } CustomData;
 
 /* CustomData.type */
@@ -115,6 +120,10 @@ typedef struct CustomData {
 #define CD_FLAG_NOFREE    (1<<1)
 /* indicates the layer is only temporary, also implies no copy */
 #define CD_FLAG_TEMPORARY ((1<<2)|CD_FLAG_NOCOPY)
+/* indicates the layer is stored in an external file */
+#define CD_FLAG_EXTERNAL  (1<<3)
+/* indicates external data is read into memory */
+#define CD_FLAG_IN_MEMORY (1<<4)
 
 /* Limits */
 #define MAX_MTFACE 8

source/blender/makesrna/intern/rna_modifier.c

@@ -340,7 +340,7 @@ static void rna_MultiresModifier_level_range(PointerRNA *ptr, int *min, int *max
 	*max = mmd->totlvl;
 }
 
-/*static int rna_MultiresModifier_external_get(PointerRNA *ptr)
+static int rna_MultiresModifier_external_get(PointerRNA *ptr)
 {
 	Object *ob= (Object*)ptr->id.data;
 	Mesh *me= ob->data;
@@ -364,7 +364,7 @@ static int rna_MultiresModifier_external_editable(PointerRNA *ptr)
 	MultiresModifierData *mmd = ptr->data;
 
 	return (G.save_over && mmd->totlvl > 0);
-}*/
+}
 
 static void modifier_object_set(Object *self, Object **ob_p, int type, PointerRNA value)
 {
@@ -570,10 +570,10 @@ static void rna_def_modifier_multires(BlenderRNA *brna)
 	RNA_def_property_clear_flag(prop, PROP_EDITABLE);
 	RNA_def_property_ui_text(prop, "Total Levels", "Number of subdivisions for which displacements are stored.");
 
-	/*prop= RNA_def_property(srna, "external", PROP_BOOLEAN, PROP_NONE);
+	prop= RNA_def_property(srna, "external", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_funcs(prop, "rna_MultiresModifier_external_get", "rna_MultiresModifier_external_set");
 	RNA_def_property_editable_func(prop, "rna_MultiresModifier_external_editable");
-	RNA_def_property_ui_text(prop, "External", "Store multires displacements outside the .blend file, to save memory.");*/
+	RNA_def_property_ui_text(prop, "External", "Store multires displacements outside the .blend file, to save memory.");
 }
 
 static void rna_def_modifier_lattice(BlenderRNA *brna)

source/blender/windowmanager/intern/wm_files.c

@@ -81,6 +81,7 @@
 #include "ED_datafiles.h"
 #include "ED_object.h"
 #include "ED_screen.h"
+#include "ED_sculpt.h"
 #include "ED_util.h"
 
 #include "GHOST_C-api.h"
@@ -492,6 +493,7 @@ void WM_write_file(bContext *C, char *target, int fileflags, ReportList *reports
 	}
 	
 	ED_object_exit_editmode(C, EM_DO_UNDO);
+	ED_sculpt_force_update(C);
 
 	do_history(di, reports);