From 0f23f618f36a7472d1c67b36344ef87a31eb586c Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 14 Feb 2018 17:33:06 +0100 Subject: [PATCH 1/6] Tests: split off render report test code from Cycles tests. This renames test environment variables from CYCLESTEST_* to BLENDER_TEST_*. Differential Revision: https://developer.blender.org/D3064 --- tests/python/cycles_render_tests.py | 390 ++----------------------- tests/python/modules/render_report.py | 398 ++++++++++++++++++++++++++ 2 files changed, 424 insertions(+), 364 deletions(-) create mode 100755 tests/python/modules/render_report.py diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 731996df8ef..a01a6f74e15 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -2,55 +2,14 @@ # Apache License, Version 2.0 import argparse -import glob import os -import pathlib import shlex import shutil import subprocess import sys -import time -import tempfile -class COLORS_ANSI: - RED = '\033[00;31m' - GREEN = '\033[00;32m' - ENDC = '\033[0m' - - -class COLORS_DUMMY: - RED = '' - GREEN = '' - ENDC = '' - -COLORS = COLORS_DUMMY - - -def print_message(message, type=None, status=''): - if type == 'SUCCESS': - print(COLORS.GREEN, end="") - elif type == 'FAILURE': - print(COLORS.RED, end="") - status_text = ... - if status == 'RUN': - status_text = " RUN " - elif status == 'OK': - status_text = " OK " - elif status == 'PASSED': - status_text = " PASSED " - elif status == 'FAILED': - status_text = " FAILED " - else: - status_text = status - if status_text: - print("[{}]" . format(status_text), end="") - print(COLORS.ENDC, end="") - print(" {}" . format(message)) - sys.stdout.flush() - - -def render_file(filepath): +def render_file(filepath, output_filepath): dirname = os.path.dirname(filepath) basedir = os.path.dirname(dirname) subject = os.path.basename(dirname) @@ -62,6 +21,8 @@ def render_file(filepath): # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] + frame_filepath = output_filepath + '0001.png' + if subject == 'opengl': command = [ BLENDER, @@ -73,7 +34,7 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", '--python', os.path.join(basedir, "util", @@ -89,7 +50,7 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", '--python', os.path.join(basedir, "util", @@ -105,321 +66,39 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", "-f", "1"] + try: + # Success output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) if VERBOSE: print(output.decode("utf-8")) return None except subprocess.CalledProcessError as e: - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) + # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(e.output.decode("utf-8")) if b"Error: engine not found" in e.output: - return "NO_CYCLES" + return "NO_ENGINE" elif b"blender probably wont start" in e.output: return "NO_START" return "CRASH" except BaseException as e: - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) + # Crash + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(e) return "CRASH" -def test_get_name(filepath): - filename = os.path.basename(filepath) - return os.path.splitext(filename)[0] - -def test_get_images(filepath): - testname = test_get_name(filepath) - dirpath = os.path.dirname(filepath) - - old_dirpath = os.path.join(dirpath, "reference_renders") - old_img = os.path.join(old_dirpath, testname + ".png") - - ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref") - ref_img = os.path.join(ref_dirpath, testname + ".png") - if not os.path.exists(ref_dirpath): - os.makedirs(ref_dirpath) - if os.path.exists(old_img): - shutil.copy(old_img, ref_img) - - new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath)) - if not os.path.exists(new_dirpath): - os.makedirs(new_dirpath) - new_img = os.path.join(new_dirpath, testname + ".png") - - diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff") - if not os.path.exists(diff_dirpath): - os.makedirs(diff_dirpath) - diff_img = os.path.join(diff_dirpath, testname + ".diff.png") - - return old_img, ref_img, new_img, diff_img - - -class Report: - def __init__(self, testname): - self.failed_tests = "" - self.passed_tests = "" - self.testname = testname - - def output(self): - # write intermediate data for single test - outdir = os.path.join(OUTDIR, self.testname) - if not os.path.exists(outdir): - os.makedirs(outdir) - - filepath = os.path.join(outdir, "failed.data") - pathlib.Path(filepath).write_text(self.failed_tests) - - filepath = os.path.join(outdir, "passed.data") - pathlib.Path(filepath).write_text(self.passed_tests) - - # gather intermediate data for all tests - failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data"))) - passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data"))) - - failed_tests = "" - passed_tests = "" - - for filename in failed_data: - filepath = os.path.join(OUTDIR, filename) - failed_tests += pathlib.Path(filepath).read_text() - for filename in passed_data: - filepath = os.path.join(OUTDIR, filename) - passed_tests += pathlib.Path(filepath).read_text() - - # write html for all tests - self.html = """ - - - Cycles Test Report - - - - -
-
-

Cycles Test Report

-
- - - - - {}{} -
NameNewReferenceDiff
-
-
- - - """ . format(failed_tests, passed_tests) - - filepath = os.path.join(OUTDIR, "report.html") - pathlib.Path(filepath).write_text(self.html) - - print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) - - def relative_url(self, filepath): - relpath = os.path.relpath(filepath, OUTDIR) - return pathlib.Path(relpath).as_posix() - - def add_test(self, filepath, error): - name = test_get_name(filepath) - name = name.replace('_', ' ') - - old_img, ref_img, new_img, diff_img = test_get_images(filepath) - - status = error if error else "" - style = """ style="background-color: #f99;" """ if error else "" - - new_url = self.relative_url(new_img) - ref_url = self.relative_url(ref_img) - diff_url = self.relative_url(diff_img) - - test_html = """ - - {}
{}
{} - - - - """ . format(style, name, self.testname, status, - new_url, ref_url, new_url, - ref_url, new_url, ref_url, - diff_url) - - if error: - self.failed_tests += test_html - else: - self.passed_tests += test_html - - -def verify_output(report, filepath): - old_img, ref_img, new_img, diff_img = test_get_images(filepath) - - # copy new image - if os.path.exists(new_img): - os.remove(new_img) - if os.path.exists(TEMP_FILE): - shutil.copy(TEMP_FILE, new_img) - - update = os.getenv('CYCLESTEST_UPDATE') - - if os.path.exists(ref_img): - # diff test with threshold - command = ( - IDIFF, - "-fail", "0.016", - "-failpercent", "1", - ref_img, - TEMP_FILE, - ) - try: - subprocess.check_output(command) - failed = False - except subprocess.CalledProcessError as e: - if VERBOSE: - print_message(e.output.decode("utf-8")) - failed = e.returncode != 1 - else: - if not update: - return False - - failed = True - - if failed and update: - # update reference - shutil.copy(new_img, ref_img) - shutil.copy(new_img, old_img) - failed = False - - # generate diff image - command = ( - IDIFF, - "-o", diff_img, - "-abs", "-scale", "16", - ref_img, - TEMP_FILE - ) - - try: - subprocess.check_output(command) - except subprocess.CalledProcessError as e: - if VERBOSE: - print_message(e.output.decode("utf-8")) - - return not failed - - -def run_test(report, filepath): - testname = test_get_name(filepath) - spacer = "." * (32 - len(testname)) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - error = render_file(filepath) - status = "FAIL" - if not error: - if not verify_output(report, filepath): - error = "VERIFY" - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_CYCLES": - print_message("Can't perform tests because Cycles failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - return error - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") - else: - print_message("Unknown error %r" % error) - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'FAILURE', 'FAILED') - return error - - - -def blend_list(path): - for dirpath, dirnames, filenames in os.walk(path): - for filename in filenames: - if filename.lower().endswith(".blend"): - filepath = os.path.join(dirpath, filename) - yield filepath - -def run_all_tests(dirpath): - passed_tests = [] - failed_tests = [] - all_files = list(blend_list(dirpath)) - all_files.sort() - report = Report(os.path.basename(dirpath)) - print_message("Running {} tests from 1 test case." . - format(len(all_files)), - 'SUCCESS', "==========") - time_start = time.time() - for filepath in all_files: - error = run_test(report, filepath) - testname = test_get_name(filepath) - if error: - if error == "NO_CYCLES": - return False - elif error == "NO_START": - return False - failed_tests.append(testname) - else: - passed_tests.append(testname) - report.add_test(filepath, error) - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - print_message("") - print_message("{} tests from 1 test case ran. ({} ms total)" . - format(len(all_files), elapsed_ms), - 'SUCCESS', "==========") - print_message("{} tests." . - format(len(passed_tests)), - 'SUCCESS', 'PASSED') - if failed_tests: - print_message("{} tests, listed below:" . - format(len(failed_tests)), - 'FAILURE', 'FAILED') - failed_tests.sort() - for test in failed_tests: - print_message("{}" . format(test), 'FAILURE', "FAILED") - - report.output() - return not bool(failed_tests) - - def create_argparse(): parser = argparse.ArgumentParser() parser.add_argument("-blender", nargs="+") @@ -433,36 +112,19 @@ def main(): parser = create_argparse() args = parser.parse_args() - global COLORS - global BLENDER, TESTDIR, IDIFF, OUTDIR - global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT - global VERBOSE - - if os.environ.get("CYCLESTEST_COLOR") is not None: - COLORS = COLORS_ANSI + global BLENDER, VERBOSE BLENDER = args.blender[0] - TESTDIR = args.testdir[0] - IDIFF = args.idiff[0] - OUTDIR = args.outdir[0] - - if not os.path.exists(OUTDIR): - os.makedirs(OUTDIR) - - TEMP = tempfile.mkdtemp() - TEMP_FILE_MASK = os.path.join(TEMP, "test") - TEMP_FILE = TEMP_FILE_MASK + "0001.png" - - TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py") - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - ok = run_all_tests(TESTDIR) + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] - # Cleanup temp files and folders - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) - os.rmdir(TEMP) + from modules import render_report + report = render_report.Report("Cycles Test Report", output_dir, idiff) + report.set_pixelated(True) + ok = report.run(test_dir, render_file) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py new file mode 100755 index 00000000000..930a08282e8 --- /dev/null +++ b/tests/python/modules/render_report.py @@ -0,0 +1,398 @@ +# Apache License, Version 2.0 +# +# Compare renders or screenshots against reference versions and generate +# a HTML report showing the differences, for regression testing. + +import glob +import os +import pathlib +import shutil +import subprocess +import sys +import time + + +class COLORS_ANSI: + RED = '\033[00;31m' + GREEN = '\033[00;32m' + ENDC = '\033[0m' + + +class COLORS_DUMMY: + RED = '' + GREEN = '' + ENDC = '' + +COLORS = COLORS_DUMMY + + +def print_message(message, type=None, status=''): + if type == 'SUCCESS': + print(COLORS.GREEN, end="") + elif type == 'FAILURE': + print(COLORS.RED, end="") + status_text = ... + if status == 'RUN': + status_text = " RUN " + elif status == 'OK': + status_text = " OK " + elif status == 'PASSED': + status_text = " PASSED " + elif status == 'FAILED': + status_text = " FAILED " + else: + status_text = status + if status_text: + print("[{}]" . format(status_text), end="") + print(COLORS.ENDC, end="") + print(" {}" . format(message)) + sys.stdout.flush() + + +def blend_list(path): + for dirpath, dirnames, filenames in os.walk(path): + for filename in filenames: + if filename.lower().endswith(".blend"): + filepath = os.path.join(dirpath, filename) + yield filepath + +def test_get_name(filepath): + filename = os.path.basename(filepath) + return os.path.splitext(filename)[0] + +def test_get_images(output_dir, filepath): + testname = test_get_name(filepath) + dirpath = os.path.dirname(filepath) + + old_dirpath = os.path.join(dirpath, "reference_renders") + old_img = os.path.join(old_dirpath, testname + ".png") + + ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") + ref_img = os.path.join(ref_dirpath, testname + ".png") + if not os.path.exists(ref_dirpath): + os.makedirs(ref_dirpath) + if os.path.exists(old_img): + shutil.copy(old_img, ref_img) + + new_dirpath = os.path.join(output_dir, os.path.basename(dirpath)) + if not os.path.exists(new_dirpath): + os.makedirs(new_dirpath) + new_img = os.path.join(new_dirpath, testname + ".png") + + diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff") + if not os.path.exists(diff_dirpath): + os.makedirs(diff_dirpath) + diff_img = os.path.join(diff_dirpath, testname + ".diff.png") + + return old_img, ref_img, new_img, diff_img + + +class Report: + __slots__ = ( + 'title', + 'output_dir', + 'idiff', + 'pixelated', + 'verbose', + 'update', + 'failed_tests', + 'passed_tests' + ) + + def __init__(self, title, output_dir, idiff): + self.title = title + self.output_dir = output_dir + self.idiff = idiff + + self.pixelated = False + self.verbose = os.environ.get("BLENDER_VERBOSE") is not None + self.update = os.getenv('BLENDER_TEST_UPDATE') is not None + + if os.environ.get("BLENDER_TEST_COLOR") is not None: + global COLORS, COLORS_ANSI + COLORS = COLORS_ANSI + + self.failed_tests = "" + self.passed_tests = "" + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + def set_pixelated(self, pixelated): + self.pixelated = pixelated + + def run(self, dirpath, render_cb): + # Run tests and output report. + dirname = os.path.basename(dirpath) + ok = self._run_all_tests(dirname, dirpath, render_cb) + self._write_html(dirname) + return ok + + def _write_html(self, dirname): + # Write intermediate data for single test. + outdir = os.path.join(self.output_dir, dirname) + if not os.path.exists(outdir): + os.makedirs(outdir) + + filepath = os.path.join(outdir, "failed.data") + pathlib.Path(filepath).write_text(self.failed_tests) + + filepath = os.path.join(outdir, "passed.data") + pathlib.Path(filepath).write_text(self.passed_tests) + + # Gather intermediate data for all tests. + failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data"))) + passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data"))) + + failed_tests = "" + passed_tests = "" + + for filename in failed_data: + filepath = os.path.join(self.output_dir, filename) + failed_tests += pathlib.Path(filepath).read_text() + for filename in passed_data: + filepath = os.path.join(self.output_dir, filename) + passed_tests += pathlib.Path(filepath).read_text() + + tests_html = failed_tests + passed_tests + + # Write html for all tests. + if self.pixelated: + image_rendering = 'pixelated' + else: + image_rendering = 'auto' + + if len(failed_tests) > 0: + message = "

Run BLENDER_TEST_UPDATE=1 ctest to create or update reference images for failed tests.

" + else: + message = "" + + html = """ + + + {title} + + + + +
+
+

{title}

+ {message} +
+ + + + + {tests_html} +
NameNewReferenceDiff
+
+
+ + + """ . format(title=self.title, + message=message, + image_rendering=image_rendering, + tests_html=tests_html) + + filepath = os.path.join(self.output_dir, "report.html") + pathlib.Path(filepath).write_text(html) + + print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) + + def _relative_url(self, filepath): + relpath = os.path.relpath(filepath, self.output_dir) + return pathlib.Path(relpath).as_posix() + + def _write_test_html(self, testname, filepath, error): + name = test_get_name(filepath) + name = name.replace('_', ' ') + + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + + status = error if error else "" + tr_style = """ style="background-color: #f99;" """ if error else "" + + new_url = self._relative_url(new_img) + ref_url = self._relative_url(ref_img) + diff_url = self._relative_url(diff_img) + + test_html = """ + + {name}
{testname}
{status} + + + + """ . format(tr_style=tr_style, + name=name, + testname=testname, + status=status, + new_url=new_url, + ref_url=ref_url, + diff_url=diff_url) + + if error: + self.failed_tests += test_html + else: + self.passed_tests += test_html + + + def _diff_output(self, filepath, tmp_filepath): + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + + # Create reference render directory. + old_dirpath = os.path.dirname(old_img) + if not os.path.exists(old_dirpath): + os.makedirs(old_dirpath) + + # Copy temporary to new image. + if os.path.exists(new_img): + os.remove(new_img) + if os.path.exists(tmp_filepath): + shutil.copy(tmp_filepath, new_img) + + if os.path.exists(ref_img): + # Diff images test with threshold. + command = ( + self.idiff, + "-fail", "0.016", + "-failpercent", "1", + ref_img, + tmp_filepath, + ) + try: + subprocess.check_output(command) + failed = False + except subprocess.CalledProcessError as e: + if self.verbose: + print_message(e.output.decode("utf-8")) + failed = e.returncode != 1 + else: + if not self.update: + return False + + failed = True + + if failed and self.update: + # Update reference image if requested. + shutil.copy(new_img, ref_img) + shutil.copy(new_img, old_img) + failed = False + + # Generate diff image. + command = ( + self.idiff, + "-o", diff_img, + "-abs", "-scale", "16", + ref_img, + tmp_filepath + ) + + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + if self.verbose: + print_message(e.output.decode("utf-8")) + + return not failed + + + def _run_test(self, filepath, render_cb): + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') + time_start = time.time() + tmp_filepath = os.path.join(self.output_dir, "tmp") + + error = render_cb(filepath, tmp_filepath) + status = "FAIL" + if not error: + if not self._diff_output(filepath, tmp_filepath): + error = "VERIFY" + + if os.path.exists(tmp_filepath): + os.remove(tmp_filepath) + + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + if not error: + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'SUCCESS', 'OK') + else: + if error == "NO_ENGINE": + print_message("Can't perform tests because the render engine failed to load!") + return error + elif error == "NO_START": + print_message('Can not perform tests because blender fails to start.', + 'Make sure INSTALL target was run.') + return error + elif error == 'VERIFY': + print_message("Rendered result is different from reference image") + else: + print_message("Unknown error %r" % error) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'FAILURE', 'FAILED') + return error + + + def _run_all_tests(self, dirname, dirpath, render_cb): + passed_tests = [] + failed_tests = [] + all_files = list(blend_list(dirpath)) + all_files.sort() + print_message("Running {} tests from 1 test case." . + format(len(all_files)), + 'SUCCESS', "==========") + time_start = time.time() + for filepath in all_files: + error = self._run_test(filepath, render_cb) + testname = test_get_name(filepath) + if error: + if error == "NO_ENGINE": + return False + elif error == "NO_START": + return False + failed_tests.append(testname) + else: + passed_tests.append(testname) + self._write_test_html(dirname, filepath, error) + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + print_message("") + print_message("{} tests from 1 test case ran. ({} ms total)" . + format(len(all_files), elapsed_ms), + 'SUCCESS', "==========") + print_message("{} tests." . + format(len(passed_tests)), + 'SUCCESS', 'PASSED') + if failed_tests: + print_message("{} tests, listed below:" . + format(len(failed_tests)), + 'FAILURE', 'FAILED') + failed_tests.sort() + for test in failed_tests: + print_message("{}" . format(test), 'FAILURE', "FAILED") + + return not bool(failed_tests) + From 3d2d58391ad8e5e0343af461d83fabe9fabd2745 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 14 Feb 2018 20:33:33 +0100 Subject: [PATCH 2/6] Tests: add OpenGL UI drawing tests. This reuses the Cycles regression test code to also work for OpenGL UI drawing. We launch Blender with a bunch of .blend files, take a screenshot and compare it with a reference screenshot, and generate a HMTL report showing the failed tests and their differences. For Cycles we keep small reference renders to compare to in svn, but for OpenGL developers currently have to generate the references manually. How to use: * WITH_OPENGL_DRAW_TESTS=ON in CMake * BLENDER_TEST_UPDATE=1 ctest -R opengl_draw * .. make code changes .. * ctest -R opengl_draw * open build_dir/tests/opengl_draw/report.html Differential Revision: https://developer.blender.org/D3064 --- CMakeLists.txt | 3 +- tests/python/CMakeLists.txt | 48 ++++++++++++- tests/python/modules/render_report.py | 11 ++- tests/python/opengl_draw_tests.py | 97 +++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 8 deletions(-) create mode 100755 tests/python/opengl_draw_tests.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b2bd2fd460..dce622d4778 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -458,7 +458,8 @@ option(WITH_BOOST "Enable features depending on boost" ON) # Unit testsing option(WITH_GTESTS "Enable GTest unit testing" OFF) -option(WITH_OPENGL_TESTS "Enable OpenGL related unit testing (Experimental)" OFF) +option(WITH_OPENGL_RENDER_TESTS "Enable OpenGL render related unit testing (Experimental)" OFF) +option(WITH_OPENGL_DRAW_TESTS "Enable OpenGL UI drawing related unit testing (Experimental)" OFF) # Documentation diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 662a35558b7..ceed2ca4d09 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -538,7 +538,7 @@ if(WITH_CYCLES) ) endif() endmacro() - if(WITH_OPENGL_TESTS) + if(WITH_OPENGL_RENDER_TESTS) add_cycles_render_test(opengl) endif() add_cycles_render_test(bake) @@ -562,6 +562,52 @@ if(WITH_CYCLES) endif() endif() +if(WITH_OPENGL_DRAW_TESTS) + if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/opengl") + macro(add_opengl_draw_test subject) + if(MSVC) + add_test( + NAME opengl_draw_${subject}_test + COMMAND + "$/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$:_d>" + ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py + -blender "$" + -testdir "${TEST_SRC_DIR}/${subject}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/opengl_draw" + ) + else() + add_test( + NAME opengl_draw_${subject}_test + COMMAND ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py + -blender "$" + -testdir "${TEST_SRC_DIR}/${subject}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/opengl_draw" + ) + endif() + endmacro() + + function(add_opengl_draw_tests) + # Use all test folders + file(GLOB children RELATIVE ${TEST_SRC_DIR} ${TEST_SRC_DIR}/*) + foreach(child ${children}) + if(IS_DIRECTORY ${TEST_SRC_DIR}/${child}) + file(GLOB blends ${TEST_SRC_DIR}/${child}/*.blend) + if(blends) + add_opengl_draw_test(${child}) + endif() + endif() + endforeach() + endfunction() + + add_opengl_draw_tests() + else() + MESSAGE(STATUS "Disabling OpenGL tests because tests folder does not exist") + endif() +endif() + + if(WITH_ALEMBIC) find_package_wrapper(Alembic) if(NOT ALEMBIC_FOUND) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 930a08282e8..5ccd5076fbc 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -49,12 +49,11 @@ def print_message(message, type=None, status=''): sys.stdout.flush() -def blend_list(path): - for dirpath, dirnames, filenames in os.walk(path): - for filename in filenames: - if filename.lower().endswith(".blend"): - filepath = os.path.join(dirpath, filename) - yield filepath +def blend_list(dirpath): + for filename in os.listdir(dirpath): + if filename.lower().endswith(".blend"): + filepath = os.path.join(dirpath, filename) + yield filepath def test_get_name(filepath): filename = os.path.basename(filepath) diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py new file mode 100755 index 00000000000..999304570df --- /dev/null +++ b/tests/python/opengl_draw_tests.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Apache License, Version 2.0 + +import argparse +import os +import shlex +import shutil +import subprocess +import sys + +def screenshot(): + import bpy + + output_path = sys.argv[-1] + + # Force redraw and take screenshot. + bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1) + bpy.ops.screen.screenshot(filepath=output_path, full=True) + + bpy.ops.wm.quit_blender() + +# When run from inside Blender, take screenshot and exit. +try: + import bpy + inside_blender = True +except ImportError: + inside_blender = False + +if inside_blender: + screenshot() + sys.exit(0) + + +def render_file(filepath, output_filepath): + command = ( + BLENDER, + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-P", + os.path.realpath(__file__), + "--", + output_filepath) + + try: + # Success + output = subprocess.check_output(command) + if VERBOSE: + print(output.decode("utf-8")) + return None + except subprocess.CalledProcessError as e: + # Error + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e.output.decode("utf-8")) + return "CRASH" + except BaseException as e: + # Crash + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e) + return "CRASH" + + +def create_argparse(): + parser = argparse.ArgumentParser() + parser.add_argument("-blender", nargs="+") + parser.add_argument("-testdir", nargs=1) + parser.add_argument("-outdir", nargs=1) + parser.add_argument("-idiff", nargs=1) + return parser + + +def main(): + parser = create_argparse() + args = parser.parse_args() + + global BLENDER, VERBOSE + + BLENDER = args.blender[0] + VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None + + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] + + from modules import render_report + report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff) + ok = report.run(test_dir, render_file) + + sys.exit(not ok) + +if __name__ == "__main__": + main() From 233a886cebb4749be49ab20adba770e8f5a41328 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Fri, 16 Feb 2018 01:22:34 +0100 Subject: [PATCH 3/6] Code cleanup: deduplicate code for running Python scripts outside Blender. --- tests/python/CMakeLists.txt | 124 ++++++++++++++---------------------- 1 file changed, 47 insertions(+), 77 deletions(-) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index ceed2ca4d09..e03d8c065e7 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -513,30 +513,34 @@ add_test( ) endif() +# Run Python script outside Blender. +function(add_python_test testname testscript) + if(MSVC) + add_test( + NAME ${testname} + COMMAND + "$/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$:_d>" + ${testscript} ${ARGN} + ) + else() + add_test( + NAME ${testname} + COMMAND ${testscript} ${ARGN} + ) + endif() +endfunction() + if(WITH_CYCLES) if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/cycles/ctests/shader") macro(add_cycles_render_test subject) - if(MSVC) - add_test( - NAME cycles_${subject}_test - COMMAND - "$/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$:_d>" - ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py - -blender "$" - -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/cycles" - ) - else() - add_test( - NAME cycles_${subject}_test - COMMAND ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py - -blender "$" - -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/cycles" - ) - endif() + add_python_test( + cycles_${subject}_test + ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py + -blender "$" + -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/cycles" + ) endmacro() if(WITH_OPENGL_RENDER_TESTS) add_cycles_render_test(opengl) @@ -564,44 +568,23 @@ endif() if(WITH_OPENGL_DRAW_TESTS) if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/opengl") - macro(add_opengl_draw_test subject) - if(MSVC) - add_test( - NAME opengl_draw_${subject}_test - COMMAND - "$/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$:_d>" + # Use all test folders + file(GLOB children RELATIVE ${TEST_SRC_DIR} ${TEST_SRC_DIR}/*) + foreach(child ${children}) + if(IS_DIRECTORY ${TEST_SRC_DIR}/${child}) + file(GLOB blends ${TEST_SRC_DIR}/${child}/*.blend) + if(blends) + add_python_test( + opengl_draw_${child}_test ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py - -blender "$" - -testdir "${TEST_SRC_DIR}/${subject}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/opengl_draw" - ) - else() - add_test( - NAME opengl_draw_${subject}_test - COMMAND ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py - -blender "$" - -testdir "${TEST_SRC_DIR}/${subject}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/opengl_draw" - ) - endif() - endmacro() - - function(add_opengl_draw_tests) - # Use all test folders - file(GLOB children RELATIVE ${TEST_SRC_DIR} ${TEST_SRC_DIR}/*) - foreach(child ${children}) - if(IS_DIRECTORY ${TEST_SRC_DIR}/${child}) - file(GLOB blends ${TEST_SRC_DIR}/${child}/*.blend) - if(blends) - add_opengl_draw_test(${child}) - endif() + -blender "$" + -testdir "${TEST_SRC_DIR}/${child}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/opengl_draw" + ) endif() - endforeach() - endfunction() - - add_opengl_draw_tests() + endif() + endforeach() else() MESSAGE(STATUS "Disabling OpenGL tests because tests folder does not exist") endif() @@ -616,26 +599,13 @@ if(WITH_ALEMBIC) get_filename_component(real_include_dir ${ALEMBIC_INCLUDE_DIR} REALPATH) get_filename_component(ALEMBIC_ROOT_DIR ${real_include_dir} DIRECTORY) - if(MSVC) - # FIXME, de-duplicate. - add_test( - NAME alembic_tests - COMMAND - "$/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$:_d>" - ${CMAKE_CURRENT_LIST_DIR}/alembic_tests.py - --blender "$" - --testdir "${TEST_SRC_DIR}/alembic" - --alembic-root "${ALEMBIC_ROOT_DIR}" - ) - else() - add_test( - NAME alembic_tests - COMMAND ${CMAKE_CURRENT_LIST_DIR}/alembic_tests.py - --blender "$" - --testdir "${TEST_SRC_DIR}/alembic" - --alembic-root "${ALEMBIC_ROOT_DIR}" - ) - endif() + add_python_test( + alembic_tests + ${CMAKE_CURRENT_LIST_DIR}/alembic_tests.py + --blender "$" + --testdir "${TEST_SRC_DIR}/alembic" + --alembic-root "${ALEMBIC_ROOT_DIR}" + ) add_test( NAME script_alembic_import From 7fdf720fb164e448703c7d9f6103a3b59178189d Mon Sep 17 00:00:00 2001 From: Gaia Clary Date: Fri, 16 Feb 2018 12:37:36 +0100 Subject: [PATCH 4/6] Fix rotation issues due to matrix to quaternion ambiguities Reviewers: mont29 Reviewed By: mont29 Subscribers: mont29 Differential Revision: https://developer.blender.org/D3066 --- source/blender/collada/AnimationImporter.cpp | 14 ++++++++-- source/blender/collada/collada_utils.cpp | 29 ++++++++++++++++++++ source/blender/collada/collada_utils.h | 1 + 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/source/blender/collada/AnimationImporter.cpp b/source/blender/collada/AnimationImporter.cpp index fec8d62933a..bc91b94afd9 100644 --- a/source/blender/collada/AnimationImporter.cpp +++ b/source/blender/collada/AnimationImporter.cpp @@ -780,6 +780,9 @@ void AnimationImporter::apply_matrix_curves(Object *ob, std::vector& a std::vector::iterator it; + float qref[4]; + unit_qt(qref); + // sample values at each frame for (it = frames.begin(); it != frames.end(); it++) { float fra = *it; @@ -815,7 +818,9 @@ void AnimationImporter::apply_matrix_curves(Object *ob, std::vector& a float rot[4], loc[3], scale[3]; - mat4_to_quat(rot, mat); + bc_rotate_from_reference_quat(rot, qref, mat); + copy_qt_qt(qref, rot); + #if 0 for (int i = 0 ; i < 4; i++) { rot[i] = RAD2DEGF(rot[i]); @@ -1190,6 +1195,9 @@ void AnimationImporter::add_bone_animation_sampled(Object *ob, std::vector::iterator it; // sample values at each frame @@ -1223,7 +1231,9 @@ void AnimationImporter::add_bone_animation_sampled(Object *ob, std::vector *objects_done, UnitConverter &unit_converter, bool scale_to_scene); extern void bc_decompose(float mat[4][4], float *loc, float eul[3], float quat[4], float *size); +extern void bc_rotate_from_reference_quat(float quat_to[4], float quat_from[4], float mat_to[4][4]); extern void bc_triangulate_mesh(Mesh *me); extern bool bc_is_leaf_bone(Bone *bone); From e1a686e44448e8345d45041631fda84d4ee0d2fa Mon Sep 17 00:00:00 2001 From: Gaia Clary Date: Fri, 16 Feb 2018 16:53:16 +0100 Subject: [PATCH 5/6] fix: limit precision also for animation matrixes if the limit option is set (gives nicer output for inspection) --- source/blender/collada/AnimationExporter.cpp | 3 +++ source/blender/collada/collada_utils.cpp | 7 +++++++ source/blender/collada/collada_utils.h | 1 + 3 files changed, 11 insertions(+) diff --git a/source/blender/collada/AnimationExporter.cpp b/source/blender/collada/AnimationExporter.cpp index d4f434d56fd..bd5cb05a1fa 100644 --- a/source/blender/collada/AnimationExporter.cpp +++ b/source/blender/collada/AnimationExporter.cpp @@ -988,6 +988,9 @@ std::string AnimationExporter::create_4x4_source(std::vector &frames, Obj double outmat[4][4]; converter.mat4_to_dae_double(outmat, mat); + if (this->export_settings->limit_precision) + bc_sanitize_mat(outmat, 6); + source.appendValues(outmat); j++; diff --git a/source/blender/collada/collada_utils.cpp b/source/blender/collada/collada_utils.cpp index c13757fa2a4..35bc643d3c7 100644 --- a/source/blender/collada/collada_utils.cpp +++ b/source/blender/collada/collada_utils.cpp @@ -864,6 +864,13 @@ void bc_sanitize_mat(float mat[4][4], int precision) mat[i][j] = double_round(mat[i][j], precision); } +void bc_sanitize_mat(double mat[4][4], int precision) +{ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + mat[i][j] = double_round(mat[i][j], precision); +} + /* * Returns name of Active UV Layer or empty String if no active UV Layer defined. * Assuming the Object is of type MESH diff --git a/source/blender/collada/collada_utils.h b/source/blender/collada/collada_utils.h index ad274777e8d..e9066d7db46 100644 --- a/source/blender/collada/collada_utils.h +++ b/source/blender/collada/collada_utils.h @@ -100,6 +100,7 @@ extern EditBone *bc_get_edit_bone(bArmature * armature, char *name); extern int bc_set_layer(int bitfield, int layer, bool enable); extern int bc_set_layer(int bitfield, int layer); extern void bc_sanitize_mat(float mat[4][4], int precision); +extern void bc_sanitize_mat(double mat[4][4], int precision); extern IDProperty *bc_get_IDProperty(Bone *bone, std::string key); extern void bc_set_IDProperty(EditBone *ebone, const char *key, float value); From 5bc2c17161cfc23ca2d8c58e7e24458c277100ae Mon Sep 17 00:00:00 2001 From: Gaia Clary Date: Fri, 16 Feb 2018 16:58:20 +0100 Subject: [PATCH 6/6] fix:T50079 collada matrix and blender matrix are transposed. This was not regarded by the animation importer, so it was creating very odd results --- source/blender/collada/AnimationImporter.cpp | 1 + source/blender/collada/collada_utils.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/blender/collada/AnimationImporter.cpp b/source/blender/collada/AnimationImporter.cpp index bc91b94afd9..95543b2dc18 100644 --- a/source/blender/collada/AnimationImporter.cpp +++ b/source/blender/collada/AnimationImporter.cpp @@ -817,6 +817,7 @@ void AnimationImporter::apply_matrix_curves(Object *ob, std::vector& a } float rot[4], loc[3], scale[3]; + transpose_m4(mat); bc_rotate_from_reference_quat(rot, qref, mat); copy_qt_qt(qref, rot); diff --git a/source/blender/collada/collada_utils.cpp b/source/blender/collada/collada_utils.cpp index 35bc643d3c7..415daccfa3d 100644 --- a/source/blender/collada/collada_utils.cpp +++ b/source/blender/collada/collada_utils.cpp @@ -403,7 +403,7 @@ void bc_rotate_from_reference_quat(float quat_to[4], float quat_from[4], float m mat4_to_quat(qd, matd); - mul_qt_qtqt(quat_to, quat_from, qd); // rot is the final rotation corresponding to mat_to + mul_qt_qtqt(quat_to, qd, quat_from); // rot is the final rotation corresponding to mat_to } void bc_triangulate_mesh(Mesh *me)