From 2ea2d98140e1a75af55d2003d0983c2fb8ff5138 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 14 Feb 2024 15:26:36 +0100 Subject: [PATCH 1/3] Tests: Add option for GPU tests on Linux --- buildbot/conf/local/machines.py | 4 +++ buildbot/pipeline/code.py | 49 +++++++++++++++++++++++++++-- buildbot/worker/blender/__init__.py | 2 ++ buildbot/worker/blender/compile.py | 12 +++++++ buildbot/worker/blender/test.py | 14 +++++++-- 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/buildbot/conf/local/machines.py b/buildbot/conf/local/machines.py index 4dddce5..0e4fb4f 100644 --- a/buildbot/conf/local/machines.py +++ b/buildbot/conf/local/machines.py @@ -5,14 +5,18 @@ _worker_names = { "code-lint": ["localhost"], "linux-x86_64-code": ["localhost"], + "linux-x86_64-code-gpu": ["localhost"], "linux-x86_64-doc-api": ["localhost"], "linux-x86_64-doc-studio-pipeline": ["localhost"], "linux-x86_64-general": ["localhost"], "linux-x86_64-store-snap": ["localhost"], "linux-x86_64-store-steam": ["localhost"], "darwin-arm64-code": ["localhost"], + "darwin-arm64-code-gpu": ["localhost"], "darwin-x86_64-code": ["localhost"], + "darwin-x86_64-code-gpu": ["localhost"], "windows-amd64-code": ["localhost"], + "windows-amd64-code-gpu": [], "windows-amd64-store-windows": ["localhost"], } diff --git a/buildbot/pipeline/code.py b/buildbot/pipeline/code.py index 58bc5dd..95adc6d 100644 --- a/buildbot/pipeline/code.py +++ b/buildbot/pipeline/code.py @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: 2011-2024 Blender Authors # +from functools import partial import pathlib +import random import buildbot.plugins @@ -133,6 +135,20 @@ scheduler_properties_common = [ strict=True, default=False, ), + buildbot.plugins.util.BooleanParameter( + name="needs_gpu_binaries", + label="GPU binaries -> build Cycles GPU kernels", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_gpu_tests", + label="GPU tests -> run EEVEE, Viewport and Cycles GPU tests", + required=True, + strict=True, + default=False, + ), ] # code-daily @@ -209,6 +225,10 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ patch_id = pipeline.common.fetch_property(props, key="patch_id", default="") override_branch_id = pipeline.common.fetch_property(props, key="override_branch_id", default="") python_module = pipeline.common.fetch_property(props, key="python_module", default=False) + needs_gpu_tests = pipeline.common.fetch_property(props, key="needs_gpu_tests", default=False) + needs_gpu_binaries = pipeline.common.fetch_property( + props, key="needs_gpu_binaries", default=False + ) build_configuration = pipeline.common.fetch_property( props, key="build_configuration", default="release" ) @@ -246,8 +266,11 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ args += ["--python-module"] if needs_full_clean: args += ["--needs-full-clean"] - if step_name in ["compile-install"] and needs_package_delivery: - args += ["--needs-gpu-binaries"] + if step_name in ["compile-gpu", "compile-install"]: + if needs_package_delivery or needs_gpu_binaries: + args += ["--needs-gpu-binaries"] + if needs_gpu_tests: + args += ["--needs-gpu-tests"] args += [step_name] @@ -380,6 +403,21 @@ def create_deliver_test_results_step(worker_config, track_id, pipeline_type): ) +def next_worker_code(worker_names_gpu, builder, workers, request): + # Use a GPU worker if needed and supported for this platform. + if not workers: + return None + + if not request.properties.getProperty("needs_gpu_tests", False) or not worker_names_gpu: + return random.choice(workers) + + for worker in workers: + if worker.worker.workername in worker_names_gpu: + return worker + + return None + + def populate(devops_env_id): builders = [] schedulers = [] @@ -478,8 +516,10 @@ def populate(devops_env_id): print(f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders") worker_group_id = f"{platform_architecture}-code" + worker_group_id_gpu = f"{platform_architecture}-code-gpu" pipeline_worker_names = platform_worker_names[worker_group_id] + pipeline_worker_names_gpu = platform_worker_names[worker_group_id_gpu] if pipeline_worker_names: # Only create the builders if the worker exists pipeline_builder_name = ( @@ -502,6 +542,7 @@ def populate(devops_env_id): buildbot.plugins.util.BuilderConfig( name=pipeline_builder_name, workernames=suitable_pipeline_worker_names, + nextWorker=partial(next_worker_code, pipeline_worker_names_gpu), tags=pipeline_builder_tags, factory=pipeline_build_factory, ) @@ -554,6 +595,8 @@ def populate(devops_env_id): "needs_package_delivery": buildbot.plugins.util.Property( "needs_package_delivery" ), + "needs_gpu_binaries": buildbot.plugins.util.Property("needs_gpu_binaries"), + "needs_gpu_tests": buildbot.plugins.util.Property("needs_gpu_tests"), "needs_skip_tests": buildbot.plugins.util.Property("needs_skip_tests"), "platform_architectures": buildbot.plugins.util.Property( "platform_architectures" @@ -637,6 +680,7 @@ def populate(devops_env_id): "python_module": False, "needs_skip_tests": False, "needs_package_delivery": False, + "needs_gpu_binaries": False, "build_configuration": "release", "platform_architectures": code_tracked_platform_architectures[track_id], } @@ -663,6 +707,7 @@ def populate(devops_env_id): "python_module": False, "needs_skip_tests": False, "needs_package_delivery": True, + "needs_gpu_binaries": True, "build_configuration": "release", "platform_architectures": code_tracked_platform_architectures[track_id], } diff --git a/buildbot/worker/blender/__init__.py b/buildbot/worker/blender/__init__.py index 1368e53..0a287d0 100644 --- a/buildbot/worker/blender/__init__.py +++ b/buildbot/worker/blender/__init__.py @@ -20,6 +20,7 @@ class CodeBuilder(worker.utils.Builder): super().__init__(args, "blender", "blender") self.needs_full_clean = args.needs_full_clean self.needs_gpu_binaries = args.needs_gpu_binaries + self.needs_gpu_tests = args.needs_gpu_tests self.needs_ninja = True self.python_module = args.python_module self.build_configuration = args.build_configuration @@ -161,6 +162,7 @@ def create_argument_parser(steps: worker.utils.BuilderSteps) -> argparse.Argumen parser = worker.utils.create_argument_parser(steps=steps) parser.add_argument("--needs-full-clean", action="store_true", required=False) parser.add_argument("--needs-gpu-binaries", action="store_true", required=False) + parser.add_argument("--needs-gpu-tests", action="store_true", required=False) parser.add_argument("--python-module", action="store_true", required=False) parser.add_argument( "--build-configuration", diff --git a/buildbot/worker/blender/compile.py b/buildbot/worker/blender/compile.py index 7add26d..8190cad 100644 --- a/buildbot/worker/blender/compile.py +++ b/buildbot/worker/blender/compile.py @@ -224,9 +224,21 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe else: options += [f"-DWITH_CYCLES_ONEAPI_BINARIES=OFF"] options += [f"-DHIP_ROOT_DIR:PATH=/opt/rocm/hip"] + + # GPU render tests support Linux + NVIDIA currently + if builder.needs_gpu_tests: + # TODO fails because missing vulkan headers? + # options += [f"-DWITH_GPU_DRAW_TESTS=ON"] + options += [f"-DWITH_GPU_RENDER_TESTS=ON"] + options += [f"-DWITH_GPU_RENDER_TESTS_SILENT=OFF"] + if builder.needs_gpu_binaries: + options += [f"-DCYCLES_TEST_DEVICES=CPU;OPTIX"] elif builder.platform == "darwin": # Metal on macOS if builder.architecture == "arm64": + if builder.needs_gpu_tests: + options += [f"-DWITH_GPU_RENDER_TESTS=ON"] + options += [f"-DWITH_GPU_RENDER_TESTS_SILENT=OFF"] options += [f"-DCYCLES_TEST_DEVICES=CPU;METAL"] if "optix" in buildbotConfig: diff --git a/buildbot/worker/blender/test.py b/buildbot/worker/blender/test.py index ef3e09e..1dafff8 100644 --- a/buildbot/worker/blender/test.py +++ b/buildbot/worker/blender/test.py @@ -18,8 +18,10 @@ def get_ctest_arguments(builder: worker.blender.CodeBuilder) -> List[str]: # Run tests in single threads for tracks which has older OpenEXR/OpenImageIO, # This prevents idiff.exe dead-lock on exit. - if builder.platform != "windows" or builder.track_id != "v330": - args += ["--parallel", "4"] + if not (builder.platform == "windows" and builder.track_id == "v330"): + # GPU tests are currently slow and can cause timeouts. + if not builder.needs_gpu_tests: + args += ["--parallel", "4"] args += ["-C", worker.blender.compile.get_cmake_build_type(builder)] return args @@ -53,8 +55,14 @@ def test(builder: worker.blender.CodeBuilder) -> None: os.chdir(builder.build_dir) success = False + env = os.environ.copy() + + # Needed to successfully initialize EGL on Rocky Linux machine. + if builder.needs_gpu_tests and builder.platform == "linux": + env["DISPLAY"] = ":0" + try: - builder.call(["ctest"] + get_ctest_arguments(builder)) + builder.call(["ctest"] + get_ctest_arguments(builder), env=env) success = True finally: package_for_upload(builder, success) -- 2.30.2 From 2a18b38873c16ae291f0debced83fa4896ecf75d Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 14 Feb 2024 15:26:36 +0100 Subject: [PATCH 2/3] Tweaks: * Upload test reports for GPU test builds only, so they are complete * Use the NVIDIA machine exclusive for GPU builds for now --- buildbot/pipeline/code.py | 24 ++++++++++++++---------- buildbot/worker/blender/test.py | 6 ++++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/buildbot/pipeline/code.py b/buildbot/pipeline/code.py index 95adc6d..8b6cb0c 100644 --- a/buildbot/pipeline/code.py +++ b/buildbot/pipeline/code.py @@ -266,7 +266,7 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ args += ["--python-module"] if needs_full_clean: args += ["--needs-full-clean"] - if step_name in ["compile-gpu", "compile-install"]: + if step_name in ["compile-gpu", "compile-install", "test-code"]: if needs_package_delivery or needs_gpu_binaries: args += ["--needs-gpu-binaries"] if needs_gpu_tests: @@ -405,17 +405,21 @@ def create_deliver_test_results_step(worker_config, track_id, pipeline_type): def next_worker_code(worker_names_gpu, builder, workers, request): # Use a GPU worker if needed and supported for this platform. - if not workers: + # GPU worker is currently reserved for GPU builds only. + compatible_workers = [] + if request.properties.getProperty("needs_gpu_tests", False) and worker_names_gpu: + for worker in workers: + if worker.worker.workername in worker_names_gpu: + compatible_workers.append(worker) + else: + for worker in workers: + if worker.worker.workername not in worker_names_gpu: + compatible_workers.append(worker) + + if not compatible_workers: return None - if not request.properties.getProperty("needs_gpu_tests", False) or not worker_names_gpu: - return random.choice(workers) - - for worker in workers: - if worker.worker.workername in worker_names_gpu: - return worker - - return None + return random.choice(compatible_workers) def populate(devops_env_id): diff --git a/buildbot/worker/blender/test.py b/buildbot/worker/blender/test.py index 1dafff8..c686d7e 100644 --- a/buildbot/worker/blender/test.py +++ b/buildbot/worker/blender/test.py @@ -43,8 +43,10 @@ def package_for_upload(builder: worker.blender.CodeBuilder, success: bool) -> No shutil.make_archive(str(package_filepath), "zip", package_tests_dir, package_filename) shutil.rmtree(package_filepath) - # Always upload unpacked folder for main and release tracks. - if builder.track_id != "vexp": + # Always upload unpacked folder for main and release tracks, + # when using GPU tests. This is useful for debugging GPU + # differences. + if builder.track_id != "vexp" and builder.needs_gpu_tests: branch = builder.branch_id.replace("blender-", "").replace("-release", "") name = f"{branch}-{builder.platform}-{builder.architecture}" shutil.copytree(build_tests_dir, package_tests_dir / name) -- 2.30.2 From 7797ddda301b57be21e06e0042ccc6ff70cb8c9d Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Fri, 16 Feb 2024 20:06:41 +0100 Subject: [PATCH 3/3] Remove unnecessary DISPLAY again --- buildbot/worker/blender/test.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/buildbot/worker/blender/test.py b/buildbot/worker/blender/test.py index c686d7e..b0fe1cf 100644 --- a/buildbot/worker/blender/test.py +++ b/buildbot/worker/blender/test.py @@ -57,14 +57,8 @@ def test(builder: worker.blender.CodeBuilder) -> None: os.chdir(builder.build_dir) success = False - env = os.environ.copy() - - # Needed to successfully initialize EGL on Rocky Linux machine. - if builder.needs_gpu_tests and builder.platform == "linux": - env["DISPLAY"] = ":0" - try: - builder.call(["ctest"] + get_ctest_arguments(builder), env=env) + builder.call(["ctest"] + get_ctest_arguments(builder)) success = True finally: package_for_upload(builder, success) -- 2.30.2