From 2b0599e7aff00f157c37fb01a124462311cb4613 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Sat, 8 Jul 2023 16:52:58 +0200 Subject: [PATCH 1/8] Realtime Compositor: add regression tests Tests must be enabled manually using the CMake flag WITH_COMPOSITOR_REALTIME_TEST. Reasons are F12 for realtime compositor is experimental and buildbots have no GPU. --- CMakeLists.txt | 1 + .../cmake/config/blender_developer.cmake | 1 + tests/python/CMakeLists.txt | 33 +++++++++++++++++-- tests/python/compositor_render_tests.py | 12 ++++++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe24ef565fa..54f686bba79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,6 +629,7 @@ endif() option(WITH_GTESTS "Enable GTest unit testing" OFF) option(WITH_OPENGL_RENDER_TESTS "Enable OpenGL render related unit testing (Experimental)" OFF) option(WITH_OPENGL_DRAW_TESTS "Enable OpenGL UI drawing related unit testing (Experimental)" OFF) +option(WITH_COMPOSITOR_REALTIME_TEST "Enable regression testing for realtime compositor" OFF) # NOTE: All callers of this must add `TEST_PYTHON_EXE_EXTRA_ARGS` before any other arguments. set(TEST_PYTHON_EXE "" CACHE PATH "Python executable to run unit tests") mark_as_advanced(TEST_PYTHON_EXE) diff --git a/build_files/cmake/config/blender_developer.cmake b/build_files/cmake/config/blender_developer.cmake index 13c9e1e75ca..8c8bcfd76d0 100644 --- a/build_files/cmake/config/blender_developer.cmake +++ b/build_files/cmake/config/blender_developer.cmake @@ -24,6 +24,7 @@ set(WITH_PYTHON_SAFETY ON CACHE BOOL "" FORCE) if(WIN32) set(WITH_WINDOWS_BUNDLE_CRT OFF CACHE BOOL "" FORCE) endif() +set(WITH_COMPOSITOR_REALTIME_TEST ON CACHE BOOL "" FORCE) # This may have issues with C++ initialization order, needs to be tested # on all platforms to be sure this is safe to enable. diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 39e79cca818..7f2612f944c 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -690,17 +690,44 @@ if(WITH_COMPOSITOR_CPU) foreach(comp_test ${compositor_tests}) add_python_test( - compositor_${comp_test}_test + compositor_${comp_test}_cpu_test ${CMAKE_CURRENT_LIST_DIR}/compositor_render_tests.py -blender "${TEST_BLENDER_EXE}" - -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" + -testdir "${TEST_SRC_DIR}/compositor_cpu/${comp_test}" -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/compositor" + -outdir "${TEST_OUT_DIR}/compositor_cpu" ) endforeach() endif() endif() +if(WITH_COMPOSITOR_REALTIME_TEST) + if(NOT OPENIMAGEIO_IDIFF) + message(WARNING "Disabling Realtime Compositor tests because OIIO idiff does not exist") + else() + set(compositor_tests + color + converter + filter + input + output + vector + + multiple_node_setups + ) + foreach(comp_test ${compositor_tests}) + add_python_test( + compositor_${comp_test}_realtime_test + ${CMAKE_CURRENT_LIST_DIR}/compositor_render_tests.py + -blender "${TEST_BLENDER_EXE}" + -testdir "${TEST_SRC_DIR}/compositor_realtime/${comp_test}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/compositor_realtime" + ) + endforeach() + endif() +endif() + set(geo_node_tests attributes curve_primitives diff --git a/tests/python/compositor_render_tests.py b/tests/python/compositor_render_tests.py index e6417aa956f..4f9e72609ac 100644 --- a/tests/python/compositor_render_tests.py +++ b/tests/python/compositor_render_tests.py @@ -20,7 +20,11 @@ except ImportError: def get_arguments(filepath, output_filepath): - return [ + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + compositor_dirname = os.path.basename(basedir) + + args = [ "--background", "-noaudio", "--factory-startup", @@ -34,6 +38,12 @@ def get_arguments(filepath, output_filepath): "-F", "PNG", "-f", "1"] + if compositor_dirname == 'compositor_realtime': + # F12 rendering with realtime compositor is experimental + args.extend(["--python-expr", "import bpy; bpy.context.preferences.experimental.use_experimental_compositors = True"]) + + return args + def create_argparse(): parser = argparse.ArgumentParser() -- 2.30.2 From fa99e79c7fb160076e8abfe6866530795592dce6 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Mon, 10 Jul 2023 00:49:31 +0200 Subject: [PATCH 2/8] Add test cases for matte and distort nodes --- tests/python/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 7f2612f944c..08c4784feaf 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -712,9 +712,15 @@ if(WITH_COMPOSITOR_REALTIME_TEST) input output vector + matte multiple_node_setups ) + + if(WITH_LIBMV) + list(APPEND compositor_tests distort) + endif() + foreach(comp_test ${compositor_tests}) add_python_test( compositor_${comp_test}_realtime_test -- 2.30.2 From d7f20b38a5047d4e0643893d0562195458377ce1 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Sun, 23 Jul 2023 21:59:26 +0200 Subject: [PATCH 3/8] Don't duplicate blend files --- tests/python/CMakeLists.txt | 16 ++++- ...ests.py => compositor_cpu_render_tests.py} | 7 +- .../compositor_realtime_render_tests.py | 72 +++++++++++++++++++ 3 files changed, 89 insertions(+), 6 deletions(-) rename tests/python/{compositor_render_tests.py => compositor_cpu_render_tests.py} (91%) create mode 100644 tests/python/compositor_realtime_render_tests.py diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 3119fd9851b..2de51837bda 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -691,13 +691,27 @@ if(WITH_COMPOSITOR_CPU) foreach(comp_test ${compositor_tests}) add_python_test( compositor_${comp_test}_cpu_test - ${CMAKE_CURRENT_LIST_DIR}/compositor_render_tests.py + ${CMAKE_CURRENT_LIST_DIR}/compositor_cpu_render_tests.py -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/compositor_cpu/${comp_test}" -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/compositor_cpu" ) endforeach() + + if(WITH_COMPOSITOR_REALTIME_TEST) + foreach(comp_test ${compositor_tests}) + add_python_test( + compositor_${comp_test}_realtime_test + ${CMAKE_CURRENT_LIST_DIR}/compositor_realtime_render_tests.py + -blender "${TEST_BLENDER_EXE}" + -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/compositor_realtime" + ) + endforeach() + endif() + endif() endif() diff --git a/tests/python/compositor_render_tests.py b/tests/python/compositor_cpu_render_tests.py similarity index 91% rename from tests/python/compositor_render_tests.py rename to tests/python/compositor_cpu_render_tests.py index 4f9e72609ac..a2e6c32595d 100644 --- a/tests/python/compositor_render_tests.py +++ b/tests/python/compositor_cpu_render_tests.py @@ -1,13 +1,10 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2015-2022 Blender Foundation +# SPDX-FileCopyrightText: 2015-2023 Blender Foundation # # SPDX-License-Identifier: Apache-2.0 import argparse import os -import shlex -import shutil -import subprocess import sys @@ -64,7 +61,7 @@ def main(): output_dir = args.outdir[0] from modules import render_report - report = render_report.Report("Compositor", output_dir, idiff) + report = render_report.Report("Compositor CPU", output_dir, idiff) report.set_pixelated(True) report.set_reference_dir("compositor_renders") diff --git a/tests/python/compositor_realtime_render_tests.py b/tests/python/compositor_realtime_render_tests.py new file mode 100644 index 00000000000..37b5a969cab --- /dev/null +++ b/tests/python/compositor_realtime_render_tests.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2015-2023 Blender Foundation +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import sys + + +# When run from inside Blender, render and exit. +try: + import bpy + inside_blender = True +except ImportError: + inside_blender = False + +NOT_IMPLEMENTED = [ + 'node_kuwahara_anisotropic.+.blend', +] + +ENABLE_REALTIME_COMPOSITOR_SCRIPT = "import bpy; " \ + "bpy.context.preferences.experimental.use_experimental_compositors = True; " \ + "bpy.data.scenes[0].node_tree.execution_mode = 'REALTIME'" + + +def get_arguments(filepath, output_filepath): + return [ + "--background", + "-noaudio", + "--factory-startup", + "--enable-autoexec", + "--debug-memory", + "--debug-exit-on-error", + filepath, + "-P", os.path.realpath(__file__), + "--python-expr", ENABLE_REALTIME_COMPOSITOR_SCRIPT, + "-o", output_filepath, + "-F", "PNG", + "-f", "1" + ] + + +def create_argparse(): + parser = argparse.ArgumentParser() + parser.add_argument("-blender", nargs="+") + parser.add_argument("-testdir", nargs=1) + parser.add_argument("-outdir", nargs=1) + parser.add_argument("-idiff", nargs=1) + return parser + + +def main(): + parser = create_argparse() + args = parser.parse_args() + + blender = args.blender[0] + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] + + from modules import render_report + report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=NOT_IMPLEMENTED) + report.set_reference_dir("compositor_realtime_renders") + + ok = report.run(test_dir, blender, get_arguments, batch=True) + + sys.exit(not ok) + + +if not inside_blender and __name__ == "__main__": + main() -- 2.30.2 From 839feed90398c1e9903752fa694608df9114a96d Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Mon, 24 Jul 2023 20:45:31 +0200 Subject: [PATCH 4/8] Remove unrelated changes --- tests/python/CMakeLists.txt | 33 ------------------- tests/python/compositor_cpu_render_tests.py | 17 +++------- .../compositor_realtime_render_tests.py | 6 +--- 3 files changed, 6 insertions(+), 50 deletions(-) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 2de51837bda..2c581b2b984 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -715,39 +715,6 @@ if(WITH_COMPOSITOR_CPU) endif() endif() -if(WITH_COMPOSITOR_REALTIME_TEST) - if(NOT OPENIMAGEIO_IDIFF) - message(WARNING "Disabling Realtime Compositor tests because OIIO idiff does not exist") - else() - set(compositor_tests - color - converter - filter - input - output - vector - matte - - multiple_node_setups - ) - - if(WITH_LIBMV) - list(APPEND compositor_tests distort) - endif() - - foreach(comp_test ${compositor_tests}) - add_python_test( - compositor_${comp_test}_realtime_test - ${CMAKE_CURRENT_LIST_DIR}/compositor_render_tests.py - -blender "${TEST_BLENDER_EXE}" - -testdir "${TEST_SRC_DIR}/compositor_realtime/${comp_test}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/compositor_realtime" - ) - endforeach() - endif() -endif() - set(geo_node_tests attributes curve_primitives diff --git a/tests/python/compositor_cpu_render_tests.py b/tests/python/compositor_cpu_render_tests.py index a2e6c32595d..dbb3cf87706 100644 --- a/tests/python/compositor_cpu_render_tests.py +++ b/tests/python/compositor_cpu_render_tests.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2015-2023 Blender Foundation +# SPDX-FileCopyrightText: 2015-2022 Blender Foundation # # SPDX-License-Identifier: Apache-2.0 import argparse import os +import shlex +import shutil +import subprocess import sys @@ -17,11 +20,7 @@ except ImportError: def get_arguments(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - compositor_dirname = os.path.basename(basedir) - - args = [ + return [ "--background", "-noaudio", "--factory-startup", @@ -35,12 +34,6 @@ def get_arguments(filepath, output_filepath): "-F", "PNG", "-f", "1"] - if compositor_dirname == 'compositor_realtime': - # F12 rendering with realtime compositor is experimental - args.extend(["--python-expr", "import bpy; bpy.context.preferences.experimental.use_experimental_compositors = True"]) - - return args - def create_argparse(): parser = argparse.ArgumentParser() diff --git a/tests/python/compositor_realtime_render_tests.py b/tests/python/compositor_realtime_render_tests.py index 37b5a969cab..d91cdd88f60 100644 --- a/tests/python/compositor_realtime_render_tests.py +++ b/tests/python/compositor_realtime_render_tests.py @@ -15,10 +15,6 @@ try: except ImportError: inside_blender = False -NOT_IMPLEMENTED = [ - 'node_kuwahara_anisotropic.+.blend', -] - ENABLE_REALTIME_COMPOSITOR_SCRIPT = "import bpy; " \ "bpy.context.preferences.experimental.use_experimental_compositors = True; " \ "bpy.data.scenes[0].node_tree.execution_mode = 'REALTIME'" @@ -60,7 +56,7 @@ def main(): output_dir = args.outdir[0] from modules import render_report - report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=NOT_IMPLEMENTED) + report = render_report.Report("Compositor Realtime", output_dir, idiff) report.set_reference_dir("compositor_realtime_renders") ok = report.run(test_dir, blender, get_arguments, batch=True) -- 2.30.2 From c97abf557c6b44c98840a50e9c9e320f8cc6c151 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Sun, 6 Aug 2023 15:33:37 +0200 Subject: [PATCH 5/8] add matte and distort test cases --- tests/python/CMakeLists.txt | 25 ++++++++++++++++--- .../compositor_realtime_render_tests.py | 15 ++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 2c581b2b984..558f976e8df 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -699,7 +699,28 @@ if(WITH_COMPOSITOR_CPU) ) endforeach() - if(WITH_COMPOSITOR_REALTIME_TEST) + endif() +endif() + +# XXX WITH_COMPOSITOR_CPU is needed for rendering. +if(WITH_COMPOSITOR_REALTIME_TEST AND WITH_COMPOSITOR_CPU) + if(NOT OPENIMAGEIO_IDIFF) + message(WARNING "Disabling realtime compositor tests because OIIO idiff does not exist") + else() + set(compositor_tests + color + converter + filter + input + output + vector + + distort + matte + + multiple_node_setups + ) + foreach(comp_test ${compositor_tests}) add_python_test( compositor_${comp_test}_realtime_test @@ -710,8 +731,6 @@ if(WITH_COMPOSITOR_CPU) -outdir "${TEST_OUT_DIR}/compositor_realtime" ) endforeach() - endif() - endif() endif() diff --git a/tests/python/compositor_realtime_render_tests.py b/tests/python/compositor_realtime_render_tests.py index d91cdd88f60..942e5626da6 100644 --- a/tests/python/compositor_realtime_render_tests.py +++ b/tests/python/compositor_realtime_render_tests.py @@ -15,6 +15,18 @@ try: except ImportError: inside_blender = False +BLACKLIST_UNSUPPORTED_RENDER_TESTS = [ + 'node_cryptomatte.blend', + 'node_cryptomatte_legacy.blend', + 'node_keying_screen.blend' +] + +BLACKLIST_CRASHING_TESTS = [ + 'node_keying.blend', + 'node_keying_edge.blend', + 'node_keying_matte.blend' +] + ENABLE_REALTIME_COMPOSITOR_SCRIPT = "import bpy; " \ "bpy.context.preferences.experimental.use_experimental_compositors = True; " \ "bpy.data.scenes[0].node_tree.execution_mode = 'REALTIME'" @@ -55,8 +67,9 @@ def main(): idiff = args.idiff[0] output_dir = args.outdir[0] + blacklist_all = BLACKLIST_CRASHING_TESTS + BLACKLIST_UNSUPPORTED_RENDER_TESTS from modules import render_report - report = render_report.Report("Compositor Realtime", output_dir, idiff) + report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=blacklist_all) report.set_reference_dir("compositor_realtime_renders") ok = report.run(test_dir, blender, get_arguments, batch=True) -- 2.30.2 From d054a0e45e479ffbd1f7383f501c19855c0ee3d1 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Fri, 11 Aug 2023 10:21:20 +0200 Subject: [PATCH 6/8] Deactivate WITH_COMPOSITOR_REALTIME_TEST for developer config --- build_files/cmake/config/blender_developer.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/build_files/cmake/config/blender_developer.cmake b/build_files/cmake/config/blender_developer.cmake index 8c8bcfd76d0..13c9e1e75ca 100644 --- a/build_files/cmake/config/blender_developer.cmake +++ b/build_files/cmake/config/blender_developer.cmake @@ -24,7 +24,6 @@ set(WITH_PYTHON_SAFETY ON CACHE BOOL "" FORCE) if(WIN32) set(WITH_WINDOWS_BUNDLE_CRT OFF CACHE BOOL "" FORCE) endif() -set(WITH_COMPOSITOR_REALTIME_TEST ON CACHE BOOL "" FORCE) # This may have issues with C++ initialization order, needs to be tested # on all platforms to be sure this is safe to enable. -- 2.30.2 From 4ff6562f5520648de91cbf5077627f67c7b66bfc Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Sat, 12 Aug 2023 11:32:30 +0100 Subject: [PATCH 7/8] add libmv as dependency to realtime compositor test + use single folder for compositor blend tests --- tests/python/CMakeLists.txt | 9 +++++---- tests/python/compositor_cpu_render_tests.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 558f976e8df..6df9d4278eb 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -693,7 +693,7 @@ if(WITH_COMPOSITOR_CPU) compositor_${comp_test}_cpu_test ${CMAKE_CURRENT_LIST_DIR}/compositor_cpu_render_tests.py -blender "${TEST_BLENDER_EXE}" - -testdir "${TEST_SRC_DIR}/compositor_cpu/${comp_test}" + -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/compositor_cpu" ) @@ -715,11 +715,12 @@ if(WITH_COMPOSITOR_REALTIME_TEST AND WITH_COMPOSITOR_CPU) output vector - distort - matte - multiple_node_setups ) + + if(WITH_LIBMV) + list(APPEND compositor_tests distort matte) + endif() foreach(comp_test ${compositor_tests}) add_python_test( diff --git a/tests/python/compositor_cpu_render_tests.py b/tests/python/compositor_cpu_render_tests.py index e3ddff918a9..1e9a4885a14 100644 --- a/tests/python/compositor_cpu_render_tests.py +++ b/tests/python/compositor_cpu_render_tests.py @@ -53,7 +53,7 @@ def main(): from modules import render_report report = render_report.Report("Compositor CPU", output_dir, idiff) report.set_pixelated(True) - report.set_reference_dir("compositor_renders") + report.set_reference_dir("compositor_cpu_renders") # Temporary change to pass OpenImageDenoise test with both 1.3 and 1.4. if os.path.basename(test_dir) == 'filter': -- 2.30.2 From 72175fcc6340fbb875714fc6d998178d236de802 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Sat, 19 Aug 2023 13:03:29 +0100 Subject: [PATCH 8/8] Replace "XXX" with "Note" --- tests/python/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 29fe4377992..f9a4693a089 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -729,7 +729,7 @@ if(WITH_COMPOSITOR_CPU) endif() endif() -# XXX WITH_COMPOSITOR_CPU is needed for rendering. +# NOTE: WITH_COMPOSITOR_CPU is needed for rendering. if(WITH_COMPOSITOR_REALTIME_TEST AND WITH_COMPOSITOR_CPU) if(NOT OPENIMAGEIO_IDIFF) message(WARNING "Disabling realtime compositor tests because OIIO idiff does not exist") -- 2.30.2