T39690: Modifications to Blender's 'temp dir' system.

Current temporary data of Blender suffers one major issue - default 'temp' dir on Windows is never
automatically cleaned up, and can end being quite big when used by Blender, especially when we have
to store per-process data (using getpid() in file names).

To address this, this patch:
* Divides tempdir paths in two, one for 'base' temp dir (the same as previous unique tempdir path),
  the other is a mkdtemp-generated sub-dir, specific to each Blender instance.
* Only uses base tempdir when we need some shallow persistance accross Blender sessions - and we always
  reuse the same filename (quit.blend...) or generate small file (crash reports...).
* Uses temp sub-dir for heavy files like pointcache or renderEXRs (Save Buffer option).
* Erases temp sub-dir on quit or crash.

To get this working it also adds a working 'recursive delete' to BLI_delete() under Windows.

Note that, as in current code, the 'recover render result' hack-feature that was possible
with SaveBuffer option is still removed. A real renderresult cache feature will be added
soon, though.

Reviewers: campbellbarton, brecht, sergey

Reviewed By: campbellbarton, sergey

CC: sergey

Differential Revision: https://developer.blender.org/D531

source/blender/blenkernel/intern/blender.c

@@ -668,7 +668,7 @@ void BKE_write_undo(bContext *C, const char *name)
 		counter = counter % U.undosteps;
 	
 		BLI_snprintf(numstr, sizeof(numstr), "%d.blend", counter);
-		BLI_make_file_string("/", filepath, BLI_temporary_dir(), numstr);
+		BLI_make_file_string("/", filepath, BLI_temp_dir_session(), numstr);
 	
 		/* success = */ /* UNUSED */ BLO_write_file(CTX_data_main(C), filepath, fileflags, NULL, NULL);
 		

source/blender/blenkernel/intern/modifier.c

@@ -709,7 +709,7 @@ const char *modifier_path_relbase(Object *ob)
 	else {
 		/* last resort, better then using "" which resolves to the current
 		 * working directory */
-		return BLI_temporary_dir();
+		return BLI_temp_dir_session();
 	}
 }
 
@@ -719,7 +719,7 @@ void modifier_path_init(char *path, int path_maxlen, const char *name)
 	/* elubie: changed this to default to the same dir as the render output
 	 * to prevent saving to C:\ on Windows */
 	BLI_join_dirfile(path, path_maxlen,
-	                 G.relbase_valid ? "//" : BLI_temporary_dir(),
+	                 G.relbase_valid ? "//" : BLI_temp_dir_session(),
 	                 name);
 }
 

source/blender/blenkernel/intern/pointcache.c

@@ -51,7 +51,6 @@
 #include "BLI_math.h"
 #include "BLI_utildefines.h"
 #include "BLI_system.h"
-#include BLI_SYSTEM_PID_H
 
 #include "BLF_translation.h"
 
@@ -96,7 +95,6 @@
 #endif
 
 /* needed for directory lookup */
-/* untitled blend's need getpid for a unique name */
 #ifndef WIN32
 #  include <dirent.h>
 #else
@@ -1466,7 +1464,7 @@ static int ptcache_path(PTCacheID *pid, char *filename)
 	
 	/* use the temp path. this is weak but better then not using point cache at all */
 	/* temporary directory is assumed to exist and ALWAYS has a trailing slash */
-	BLI_snprintf(filename, MAX_PTCACHE_PATH, "%s"PTCACHE_PATH"%d", BLI_temporary_dir(), abs(getpid()));
+	BLI_snprintf(filename, MAX_PTCACHE_PATH, "%s"PTCACHE_PATH, BLI_temp_dir_session());
 	
 	return BLI_add_slash(filename); /* new strlen() */
 }

source/blender/blenkernel/intern/smoke.c

@@ -205,7 +205,7 @@ void smoke_reallocate_highres_fluid(SmokeDomainSettings *sds, float dx, int res[
 	/* smoke_turbulence_init uses non-threadsafe functions from fftw3 lib (like fftw_plan & co). */
 	BLI_lock_thread(LOCK_FFTW);
 
-	sds->wt = smoke_turbulence_init(res, sds->amplify + 1, sds->noise, BLI_temporary_dir(), use_fire, use_colors);
+	sds->wt = smoke_turbulence_init(res, sds->amplify + 1, sds->noise, BLI_temp_dir_session(), use_fire, use_colors);
 
 	BLI_unlock_thread(LOCK_FFTW);
 

source/blender/blenlib/BLI_path_util.h

@@ -191,12 +191,14 @@ void BLI_char_switch(char *string, char from, char to) ATTR_NONNULL();
 void BLI_init_program_path(const char *argv0);
 /* Initialize path to temporary directory.
  * NOTE: On Window userdir will be set to the temporary directory! */
-void BLI_init_temporary_dir(char *userdir);
+void BLI_temp_dir_init(char *userdir);
 
 const char *BLI_program_path(void);
 const char *BLI_program_dir(void);
-const char *BLI_temporary_dir(void);
+const char *BLI_temp_dir_session(void);
+const char *BLI_temp_dir_base(void);
 void BLI_system_temporary_dir(char *dir);
+void BLI_temp_dir_session_purge(void);
 
 #ifdef WITH_ICONV
 void BLI_string_to_utf8(char *original, char *utf_8, const char *code);

source/blender/blenlib/intern/fileops.c

@@ -48,6 +48,7 @@
 #  include <io.h>
 #  include "BLI_winstuff.h"
 #  include "BLI_callbacks.h"
+#  include "BLI_fileops_types.h"
 #  include "utf_winfunc.h"
 #  include "utfconv.h"
 #else
@@ -284,26 +285,72 @@ int   BLI_access(const char *filename, int mode)
 	return uaccess(filename, mode);
 }
 
-int BLI_delete(const char *file, bool dir, bool recursive)
+static bool delete_unique(const char *path, const bool dir)
 {
-	int err;
+	bool err;
-	
-	UTF16_ENCODE(file);
 
-	if (recursive) {
+	UTF16_ENCODE(path);
-		callLocalErrorCallBack("Recursive delete is unsupported on Windows");
+
-		err = 1;
+	if (dir) {
-	}
+		err = !RemoveDirectoryW(path_16);
-	else if (dir) {
-		err = !RemoveDirectoryW(file_16);
 		if (err) printf("Unable to remove directory");
 	}
 	else {
-		err = !DeleteFileW(file_16);
+		err = !DeleteFileW(path_16);
 		if (err) callLocalErrorCallBack("Unable to delete file");
 	}
 
-	UTF16_UN_ENCODE(file);
+	UTF16_UN_ENCODE(path);
+
+	return err;
+}
+
+static bool delete_recursive(const char *dir)
+{
+	struct direntry *filelist, *fl;
+	bool err = false;
+	unsigned int nbr, i;
+
+	i = nbr = BLI_dir_contents(dir, &filelist);
+	fl = filelist;
+	while(i--) {
+		char file[8];
+		BLI_split_file_part(fl->path, file, sizeof(file));
+		if (STREQ(file, ".") || STREQ(file, "..")) {
+			/* Skip! */
+		}
+		else if (S_ISDIR(fl->type)) {
+			if (delete_recursive(fl->path) {
+				err = true;
+			}
+		}
+		else {
+			if (delete_unique(fl->path, false)) {
+				err = true;
+			}
+		}
+		++fl;
+	}
+
+	if (!err && delete_unique(dir, true)) {
+		err = true;
+	}
+
+	BLI_free_filelist(filelist, nbr);
+
+	return err;
+}
+
+int BLI_delete(const char *file, bool dir, bool recursive)
+{
+	int err;
+
+	if (recursive) {
+		err = delete_recursive(file);
+	}
+	else {
+		err = delete_unique(file, dir);
+	}
 
 	return err;
 }

source/blender/blenlib/intern/path_util.c

@@ -49,9 +49,9 @@
 
 #include "GHOST_Path-api.h"
 
-#ifdef WIN32
+#include "MEM_guardedalloc.h"
-#  include "MEM_guardedalloc.h"
 
+#ifdef WIN32
 #  include "utf_winfunc.h"
 #  include "utfconv.h"
 #  include <io.h>
@@ -73,7 +73,8 @@
 
 static char bprogname[FILE_MAX];    /* full path to program executable */
 static char bprogdir[FILE_MAX];     /* full path to directory in which executable is located */
-static char btempdir[FILE_MAX];     /* temporary directory */
+static char btempdir_base[FILE_MAX];          /* persistent temporary directory */
+static char btempdir_session[FILE_MAX] = "";  /* volatile temporary directory */
 
 /* implementation */
 
@@ -2319,14 +2320,21 @@ const char *BLI_program_dir(void)
  * 
  * Also make sure the temp dir has a trailing slash
  *
- * \param fullname The full path to the temp directory
+ * \param fullname The full path to the temporary temp directory
+ * \param basename The full path to the persistent temp directory (may be NULL)
  * \param maxlen The size of the fullname buffer
  * \param userdir Directory specified in user preferences 
  */
-static void BLI_where_is_temp(char *fullname, const size_t maxlen, char *userdir)
+static void BLI_where_is_temp(char *fullname, char *basename, const size_t maxlen, char *userdir)
 {
+	/* Clear existing temp dir, if needed. */
+	BLI_temp_dir_session_purge();
+
 	fullname[0] = '\0';
-	
+	if (basename) {
+		basename[0] = '\0';
+	}
+
 	if (userdir && BLI_is_dir(userdir)) {
 		BLI_strncpy(fullname, userdir, maxlen);
 	}
@@ -2368,23 +2376,59 @@ static void BLI_where_is_temp(char *fullname, const size_t maxlen, char *userdir
 		}
 #endif
 	}
+
+	/* Now that we have a valid temp dir, add system-generated unique sub-dir. */
+	if (basename) {
+		/* 'XXXXXX' is kind of tag to be replaced by mktemp-familly by an uuid. */
+		char *tmp_name = BLI_strdupcat(fullname, "blender_XXXXXX");
+		const size_t ln = strlen(tmp_name) + 1;
+		if (ln <= maxlen) {
+#ifdef WIN32
+			if (_mktemp_s(tmp_name, ln) == 0) {
+				BLI_dir_create_recursive(tmp_name);
+			}
+#else
+			mkdtemp(tmp_name);
+#endif
+		}
+		if (BLI_is_dir(tmp_name)) {
+			BLI_strncpy(basename, fullname, maxlen);
+			BLI_strncpy(fullname, tmp_name, maxlen);
+			BLI_add_slash(fullname);
+		}
+		else {
+			printf("Warning! Could not generate a temp file name for '%s', falling back to '%s'\n", tmp_name, fullname);
+		}
+
+		MEM_freeN(tmp_name);
+	}
 }
 
 /**
- * Sets btempdir to userdir if specified and is a valid directory, otherwise
+ * Sets btempdir_base to userdir if specified and is a valid directory, otherwise
  * chooses a suitable OS-specific temporary directory.
+ * Sets btempdir_session to a mkdtemp-generated sub-dir of btempdir_base.
  */
-void BLI_init_temporary_dir(char *userdir)
+void BLI_temp_dir_init(char *userdir)
 {
-	BLI_where_is_temp(btempdir, FILE_MAX, userdir);
+	BLI_where_is_temp(btempdir_session, btempdir_base, FILE_MAX, userdir);
+;
 }
 
 /**
  * Path to temporary directory (with trailing slash)
  */
-const char *BLI_temporary_dir(void)
+const char *BLI_temp_dir_session(void)
 {
-	return btempdir;
+	return btempdir_session[0] ? btempdir_session : BLI_temp_dir_base();
+}
+
+/**
+ * Path to persistent temporary directory (with trailing slash)
+ */
+const char *BLI_temp_dir_base(void)
+{
+	return btempdir_base;
 }
 
 /**
@@ -2392,7 +2436,17 @@ const char *BLI_temporary_dir(void)
  */
 void BLI_system_temporary_dir(char *dir)
 {
-	BLI_where_is_temp(dir, FILE_MAX, NULL);
+	BLI_where_is_temp(dir, NULL, FILE_MAX, NULL);
+}
+
+/**
+ * Delete content of this instance's temp dir.
+ */
+void BLI_temp_dir_session_purge(void)
+{
+	if (btempdir_session[0] && BLI_is_dir(btempdir_session)) {
+		BLI_delete(btempdir_session, true, true);
+	}
 }
 
 #ifdef WITH_ICONV

source/blender/compositor/intern/COM_Debug.cpp

@@ -398,7 +398,7 @@ void DebugInfo::graphviz(const ExecutionSystem *system)
 		char filename[FILE_MAX];
 		
 		BLI_snprintf(basename, sizeof(basename), "compositor_%d.dot", m_file_index);
-		BLI_join_dirfile(filename, sizeof(filename), BLI_temporary_dir(), basename);
+		BLI_join_dirfile(filename, sizeof(filename), BLI_temp_dir_session(), basename);
 		++m_file_index;
 		
 		FILE *fp = BLI_fopen(filename, "wb");

source/blender/editors/space_view3d/view3d_ops.c

@@ -77,7 +77,7 @@ static int view3d_copybuffer_exec(bContext *C, wmOperator *op)
 	}
 	CTX_DATA_END;
 	
-	BLI_make_file_string("/", str, BLI_temporary_dir(), "copybuffer.blend");
+	BLI_make_file_string("/", str, BLI_temp_dir_session(), "copybuffer.blend");
 	BKE_copybuffer_save(str, op->reports);
 	
 	BKE_report(op->reports, RPT_INFO, "Copied selected objects to buffer");
@@ -102,7 +102,7 @@ static int view3d_pastebuffer_exec(bContext *C, wmOperator *op)
 {
 	char str[FILE_MAX];
 
-	BLI_make_file_string("/", str, BLI_temporary_dir(), "copybuffer.blend");
+	BLI_make_file_string("/", str, BLI_temp_dir_session(), "copybuffer.blend");
 	if (BKE_copybuffer_paste(C, str, op->reports)) {
 		WM_event_add_notifier(C, NC_WINDOW, NULL);
 

source/blender/makesrna/intern/rna_userdef.c

@@ -386,7 +386,7 @@ static void rna_userdef_pathcompare_remove(ReportList *reports, PointerRNA *path
 
 static void rna_userdef_temp_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *UNUSED(ptr))
 {
-	BLI_init_temporary_dir(U.tempdir);
+	BLI_temp_dir_init(U.tempdir);
 }
 
 static void rna_userdef_text_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *UNUSED(ptr))

source/blender/python/intern/bpy_app.c

@@ -250,7 +250,7 @@ PyDoc_STRVAR(bpy_app_tempdir_doc,
 );
 static PyObject *bpy_app_tempdir_get(PyObject *UNUSED(self), void *UNUSED(closure))
 {
-	return PyC_UnicodeFromByte(BLI_temporary_dir());
+	return PyC_UnicodeFromByte(BLI_temp_dir_session());
 }
 
 PyDoc_STRVAR(bpy_app_driver_dict_doc,

source/blender/render/intern/source/render_result.c

@@ -41,7 +41,6 @@
 #include "BLI_rect.h"
 #include "BLI_string.h"
 #include "BLI_system.h"
-#include BLI_SYSTEM_PID_H
 #include "BLI_threads.h"
 
 #include "BKE_image.h"
@@ -1030,14 +1029,13 @@ void render_result_exr_file_path(Scene *scene, const char *layname, int sample,
 	
 	BLI_split_file_part(G.main->name, fi, sizeof(fi));
 	if (sample == 0) {
-		BLI_snprintf(name, sizeof(name), "%s_%s_%s_%d.exr", fi, scene->id.name + 2, layname, abs(getpid()));
+		BLI_snprintf(name, sizeof(name), "%s_%s_%s.exr", fi, scene->id.name + 2, layname);
 	}
 	else {
-		BLI_snprintf(name, sizeof(name), "%s_%s_%s%d_%d.exr", fi, scene->id.name + 2, layname, sample,
+		BLI_snprintf(name, sizeof(name), "%s_%s_%s%d.exr", fi, scene->id.name + 2, layname, sample);
-		             abs(getpid()));
 	}
 
-	BLI_make_file_string("/", filepath, BLI_temporary_dir(), name);
+	BLI_make_file_string("/", filepath, BLI_temp_dir_session(), name);
 }
 
 /* only for temp buffer files, makes exact copy of render result */

source/blender/windowmanager/intern/wm_files.c

@@ -299,7 +299,7 @@ static void wm_init_userdef(bContext *C, const bool from_memory)
 	}
 
 	/* update tempdir from user preferences */
-	BLI_init_temporary_dir(U.tempdir);
+	BLI_temp_dir_init(U.tempdir);
 
 	BKE_userdef_state();
 }
@@ -591,7 +591,7 @@ int wm_homefile_read(bContext *C, ReportList *reports, bool from_memory, const c
 		if (BLI_listbase_is_empty(&wmbase)) {
 			wm_clear_default_size(C);
 		}
-		BLI_init_temporary_dir(U.tempdir);
+		BLI_temp_dir_init(U.tempdir);
 
 #ifdef WITH_PYTHON_SECURITY
 		/* use alternative setting for security nuts
@@ -1058,14 +1058,14 @@ void wm_autosave_location(char *filepath)
 	 * BLI_make_file_string will create string that has it most likely on C:\
 	 * through get_default_root().
 	 * If there is no C:\tmp autosave fails. */
-	if (!BLI_exists(BLI_temporary_dir())) {
+	if (!BLI_exists(BLI_temp_dir_base())) {
 		savedir = BLI_get_folder_create(BLENDER_USER_AUTOSAVE, NULL);
 		BLI_make_file_string("/", filepath, savedir, pidstr);
 		return;
 	}
 #endif
 
-	BLI_make_file_string("/", filepath, BLI_temporary_dir(), pidstr);
+	BLI_make_file_string("/", filepath, BLI_temp_dir_base(), pidstr);
 }
 
 void WM_autosave_init(wmWindowManager *wm)
@@ -1129,7 +1129,7 @@ void wm_autosave_delete(void)
 
 	if (BLI_exists(filename)) {
 		char str[FILE_MAX];
-		BLI_make_file_string("/", str, BLI_temporary_dir(), BLENDER_QUIT_FILE);
+		BLI_make_file_string("/", str, BLI_temp_dir_base(), BLENDER_QUIT_FILE);
 
 		/* if global undo; remove tempsave, otherwise rename */
 		if (U.uiflag & USER_GLOBALUNDO) BLI_delete(filename, false, false);

source/blender/windowmanager/intern/wm_init_exit.c

@@ -405,7 +405,7 @@ void WM_exit_ext(bContext *C, const bool do_python)
 				/* save the undo state as quit.blend */
 				char filename[FILE_MAX];
 				
-				BLI_make_file_string("/", filename, BLI_temporary_dir(), BLENDER_QUIT_FILE);
+				BLI_make_file_string("/", filename, BLI_temp_dir_base(), BLENDER_QUIT_FILE);
 
 				if (BKE_undo_save_file(filename))
 					printf("Saved session recovery to '%s'\n", filename);
@@ -527,6 +527,8 @@ void WM_exit_ext(bContext *C, const bool do_python)
 		MEM_printmemlist();
 	}
 	wm_autosave_delete();
+
+	BLI_temp_dir_session_purge();
 }
 
 void WM_exit(bContext *C)

source/blender/windowmanager/intern/wm_operators.c

@@ -2587,7 +2587,7 @@ void WM_recover_last_session(bContext *C, ReportList *reports)
 {
 	char filepath[FILE_MAX];
 	
-	BLI_make_file_string("/", filepath, BLI_temporary_dir(), BLENDER_QUIT_FILE);
+	BLI_make_file_string("/", filepath, BLI_temp_dir_base(), BLENDER_QUIT_FILE);
 	/* if reports==NULL, it's called directly without operator, we add a quick check here */
 	if (reports || BLI_exists(filepath)) {
 		G.fileflags |= G_FILE_RECOVER;

source/creator/creator.c

@@ -566,7 +566,7 @@ static void blender_crash_handler(int signum)
 		char fname[FILE_MAX];
 
 		if (!G.main->name[0]) {
-			BLI_make_file_string("/", fname, BLI_temporary_dir(), "crash.blend");
+			BLI_make_file_string("/", fname, BLI_temp_dir_base(), "crash.blend");
 		}
 		else {
 			BLI_strncpy(fname, G.main->name, sizeof(fname));
@@ -587,10 +587,10 @@ static void blender_crash_handler(int signum)
 	char fname[FILE_MAX];
 
 	if (!G.main->name[0]) {
-		BLI_join_dirfile(fname, sizeof(fname), BLI_temporary_dir(), "blender.crash.txt");
+		BLI_join_dirfile(fname, sizeof(fname), BLI_temp_dir_base(), "blender.crash.txt");
 	}
 	else {
-		BLI_join_dirfile(fname, sizeof(fname), BLI_temporary_dir(), BLI_path_basename(G.main->name));
+		BLI_join_dirfile(fname, sizeof(fname), BLI_temp_dir_base(), BLI_path_basename(G.main->name));
 		BLI_replace_extension(fname, sizeof(fname), ".crash.txt");
 	}
 
@@ -621,6 +621,8 @@ static void blender_crash_handler(int signum)
 		fclose(fp);
 	}
 
+	/* Delete content of temp dir! */
+	BLI_temp_dir_session_purge();
 
 	/* really crash */
 	signal(signum, SIG_DFL);
@@ -1666,7 +1668,7 @@ int main(
 
 		/* this is properly initialized with user defs, but this is default */
 		/* call after loading the startup.blend so we can read U.tempdir */
-		BLI_init_temporary_dir(U.tempdir);
+		BLI_temp_dir_init(U.tempdir);
 	}
 	else {
 #ifndef WITH_PYTHON_MODULE
@@ -1676,7 +1678,7 @@ int main(
 		WM_init(C, argc, (const char **)argv);
 
 		/* don't use user preferences temp dir */
-		BLI_init_temporary_dir(NULL);
+		BLI_temp_dir_init(NULL);
 	}
 #ifdef WITH_PYTHON
 	/**

source/gameengine/GamePlayer/ghost/GPG_ghost.cpp

@@ -430,7 +430,7 @@ int main(int argc, char** argv)
 #endif /* __alpha__ */
 #endif /* __linux__ */
 	BLI_init_program_path(argv[0]);
-	BLI_init_temporary_dir(NULL);
+	BLI_temp_dir_init(NULL);
 	
 	// We don't use threads directly in the BGE, but we need to call this so things like
 	// freeing up GPU_Textures works correctly.
@@ -1142,5 +1142,7 @@ int main(int argc, char** argv)
 		MEM_printmemlist();
 	}
 
+	BLI_temp_dir_session_purge();
+
 	return error ? -1 : 0;
 }