Tests: speed up render tests by running multiple in the same process

Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process. If a test crashes, the remaining tests in the same category will
be marked as skipped.

Benchmarked on a quad core with ctest -j8.

cycles: 118.1s -> 94.3s
eevee: 66.2s -> 29.2s
workbench: 31.7s -> 8.6s

tests/python/cycles_render_tests.py

@@ -9,77 +9,76 @@ import subprocess
 import sys
 
 
-def render_file(filepath, output_filepath):
+def render_files(filepaths, output_filepaths):
-    dirname = os.path.dirname(filepath)
+    command = [BLENDER, "--background"]
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
-    common_args = [
-        "-noaudio",
-        "--factory-startup",
-        "--enable-autoexec",
-        filepath,
-        "-E", "CYCLES",
-        "-o", output_filepath,
-        "-F", "PNG"]
 
     # OSL and GPU examples
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
     custom_args = os.getenv('CYCLESTEST_ARGS')
     custom_args = shlex.split(custom_args) if custom_args else []
-    common_args += custom_args
 
-    if subject == 'opengl':
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
-        command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
+        dirname = os.path.dirname(filepath)
-        command += common_args
+        basedir = os.path.dirname(dirname)
-        command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
+        subject = os.path.basename(dirname)
-    elif subject == 'bake':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
-    elif subject == 'denoise_animation':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
-    else:
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ["-f", "1"]
 
+        frame_filepath = output_filepath + '0001.png'
+
+        common_args = [
+            "-noaudio",
+            "--factory-startup",
+            "--enable-autoexec",
+            filepath,
+            "-E", "CYCLES",
+            "-o", output_filepath,
+            "-F", "PNG"]
+
+        common_args += custom_args
+
+        if subject == 'bake':
+            command.extend(common_args)
+            command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
+        elif subject == 'denoise_animation':
+            command.extend(common_args)
+            command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
+        else:
+            command.extend(common_args)
+            command.extend(["-f", "1"])
+
+    error = None
     try:
         # Success
         output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
-        return None
     except subprocess.CalledProcessError as e:
         # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
+        error = "CRASH"
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
     except BaseException as e:
         # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
-            print(e)
+            print(e.decode("utf-8"))
-        return "CRASH"
+        error = "CRASH"
 
+    # Detect missing filepaths and consider those errors
+    errors = []
+    for output_filepath in output_filepaths:
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+            errors.append(None)
+        else:
+            errors.append(error)
+            error = 'SKIPPED'
+
+    return errors
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -108,7 +107,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("cycles_renders")
     report.set_compare_engines('cycles', 'eevee')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)
 

tests/python/eevee_render_tests.py

@@ -49,57 +49,61 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
+def render_files(filepaths, output_filepaths):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
     command = [
         BLENDER,
         "--background",
         "-noaudio",
         "--factory-startup",
-        "--enable-autoexec",
+        "--enable-autoexec"]
-        filepath,
-        "-E", "BLENDER_EEVEE",
-        "-P",
-        os.path.realpath(__file__),
-        "-o", output_filepath,
-        "-F", "PNG",
-        "-f", "1"]
 
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+
+        command.extend([
+            filepath,
+            "-E", "BLENDER_EEVEE",
+            "-P",
+            os.path.realpath(__file__),
+            "-o", output_filepath,
+            "-F", "PNG",
+            "-f", "1"])
+
+    error = None
     try:
         # Success
         output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
-        return None
     except subprocess.CalledProcessError as e:
         # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
+        error = "CRASH"
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
     except BaseException as e:
         # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
-            print(e)
+            print(e.decode("utf-8"))
-        return "CRASH"
+        error = "CRASH"
+
+    # Detect missing filepaths and consider those errors
+    errors = []
+    for output_filepath in output_filepaths:
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+            errors.append(None)
+        else:
+            errors.append(error)
+            error = 'SKIPPED'
+
+    return errors
 
 
 def create_argparse():
@@ -129,7 +133,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("eevee_renders")
     report.set_compare_engines('eevee', 'cycles')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)
 

tests/python/modules/render_report.py

@@ -374,41 +374,48 @@ class Report:
 
         return not failed
 
-    def _run_test(self, filepath, render_cb):
+    def _run_tests(self, filepaths, render_cb):
-        testname = test_get_name(filepath)
+        # Run all tests together for performance, since Blender
-        print_message(testname, 'SUCCESS', 'RUN')
+        # startup time is a significant factor.
-        time_start = time.time()
+        tmp_filepaths = []
-        tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+        for filepath in filepaths:
+            testname = test_get_name(filepath)
+            print_message(testname, 'SUCCESS', 'RUN')
+            tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname))
 
-        error = render_cb(filepath, tmp_filepath)
+        run_errors = render_cb(filepaths, tmp_filepaths)
-        status = "FAIL"
+        errors = []
-        if not error:
-            if not self._diff_output(filepath, tmp_filepath):
-                error = "VERIFY"
 
-        if os.path.exists(tmp_filepath):
+        for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths):
-            os.remove(tmp_filepath)
+            if not error:
+                if os.path.getsize(tmp_filepath) == 0:
+                    error = "VERIFY"
+                elif not self._diff_output(filepath, tmp_filepath):
+                    error = "VERIFY"
 
-        time_end = time.time()
+            if os.path.exists(tmp_filepath):
-        elapsed_ms = int((time_end - time_start) * 1000)
+                os.remove(tmp_filepath)
-        if not error:
+
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
+            errors.append(error)
-                          'SUCCESS', 'OK')
+
-        else:
+            testname = test_get_name(filepath)
-            if error == "NO_ENGINE":
+            if not error:
-                print_message("Can't perform tests because the render engine failed to load!")
+                print_message(testname, 'SUCCESS', 'OK')
-                return error
-            elif error == "NO_START":
-                print_message('Can not perform tests because blender fails to start.',
-                              'Make sure INSTALL target was run.')
-                return error
-            elif error == 'VERIFY':
-                print_message("Rendered result is different from reference image")
             else:
-                print_message("Unknown error %r" % error)
+                if error == "SKIPPED":
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
+                    print_message("Skipped after previous render caused error")
-                          'FAILURE', 'FAILED')
+                elif error == "NO_ENGINE":
-        return error
+                    print_message("Can't perform tests because the render engine failed to load!")
+                elif error == "NO_START":
+                    print_message('Can not perform tests because blender fails to start.',
+                                  'Make sure INSTALL target was run.')
+                elif error == 'VERIFY':
+                    print_message("Rendered result is different from reference image")
+                else:
+                    print_message("Unknown error %r" % error)
+                print_message(testname, 'FAILURE', 'FAILED')
+
+        return errors
 
     def _run_all_tests(self, dirname, dirpath, render_cb):
         passed_tests = []
@@ -419,8 +426,8 @@ class Report:
                       format(len(all_files)),
                       'SUCCESS', "==========")
         time_start = time.time()
-        for filepath in all_files:
+        errors = self._run_tests(all_files, render_cb)
-            error = self._run_test(filepath, render_cb)
+        for filepath, error in zip(all_files, errors):
             testname = test_get_name(filepath)
             if error:
                 if error == "NO_ENGINE":

tests/python/opengl_draw_tests.py

@@ -31,41 +31,48 @@ if inside_blender:
     sys.exit(0)
 
 
-def render_file(filepath, output_filepath):
+def render_files(filepaths, output_filepaths):
-    command = (
+    errors = []
-        BLENDER,
-        "--no-window-focus",
-        "--window-geometry",
-        "0", "0", "1024", "768",
-        "-noaudio",
-        "--factory-startup",
-        "--enable-autoexec",
-        filepath,
-        "-P",
-        os.path.realpath(__file__),
-        "--",
-        output_filepath)
 
-    try:
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
-        # Success
+        command = (
-        output = subprocess.check_output(command)
+            BLENDER,
-        if VERBOSE:
+            "--no-window-focus",
-            print(output.decode("utf-8"))
+            "--window-geometry",
-        return None
+            "0", "0", "1024", "768",
-    except subprocess.CalledProcessError as e:
+            "-noaudio",
-        # Error
+            "--factory-startup",
-        if os.path.exists(output_filepath):
+            "--enable-autoexec",
-            os.remove(output_filepath)
+            filepath,
-        if VERBOSE:
+            "-P",
-            print(e.output.decode("utf-8"))
+            os.path.realpath(__file__),
-        return "CRASH"
+            "--",
-    except BaseException as e:
+            output_filepath)
-        # Crash
+
-        if os.path.exists(output_filepath):
+        error = None
-            os.remove(output_filepath)
+        try:
-        if VERBOSE:
+            # Success
-            print(e)
+            output = subprocess.check_output(command)
-        return "CRASH"
+            if VERBOSE:
+                print(output.decode("utf-8"))
+        except subprocess.CalledProcessError as e:
+            # Error
+            if os.path.exists(output_filepath):
+                os.remove(output_filepath)
+            if VERBOSE:
+                print(e.output.decode("utf-8"))
+            error = "CRASH"
+        except BaseException as e:
+            # Crash
+            if os.path.exists(output_filepath):
+                os.remove(output_filepath)
+            if VERBOSE:
+                print(e)
+            error = "CRASH"
+
+        errors.append(error)
+
+    return errors
 
 
 def create_argparse():
@@ -92,7 +99,7 @@ def main():
 
     from modules import render_report
     report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff)
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)
 

tests/python/workbench_render_tests.py

@@ -34,57 +34,61 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
+def render_files(filepaths, output_filepaths):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
     command = [
         BLENDER,
         "--background",
         "-noaudio",
         "--factory-startup",
-        "--enable-autoexec",
+        "--enable-autoexec"]
-        filepath,
-        "-E", "BLENDER_WORKBENCH",
-        "-P",
-        os.path.realpath(__file__),
-        "-o", output_filepath,
-        "-F", "PNG",
-        "-f", "1"]
 
+    for filepath, output_filepath in zip(filepaths, output_filepaths):
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+
+        command.extend([
+            filepath,
+            "-E", "BLENDER_WORKBENCH",
+            "-P",
+            os.path.realpath(__file__),
+            "-o", output_filepath,
+            "-F", "PNG",
+            "-f", "1"])
+
+    error = None
     try:
         # Success
         output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
-        return None
     except subprocess.CalledProcessError as e:
         # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
+        error = "CRASH"
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
     except BaseException as e:
         # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
-            print(e)
+            print(e.decode("utf-8"))
-        return "CRASH"
+        error = "CRASH"
+
+    # Detect missing filepaths and consider those errors
+    errors = []
+    for output_filepath in output_filepaths:
+        frame_filepath = output_filepath + '0001.png'
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+            errors.append(None)
+        else:
+            errors.append(error)
+            error = 'SKIPPED'
+
+    return errors
 
 
 def create_argparse():
@@ -114,7 +118,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("workbench_renders")
     report.set_compare_engines('workbench', 'eevee')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, render_files)
 
     sys.exit(not ok)