infrastructure/blender-devops
infrastructure/blender-devops
1
1
Fork 8
Code Pull Requests 3 Activity

WIP: Tests: Add option to run GPU tests on buildbot #2

Closed
Brecht Van Lommel wants to merge 3 commits from gpu-tests into main
pull from: gpu-tests
merge into: infrastructure:main

When changing the target branch, be careful to rebase the branch in your fork to match. See documentation.
Conversation 3 Commits 3 Files Changed 5 +77 -6
5 changed files with 77 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
buildbot/conf/local/machines.py
View File

@ -5,14 +5,18 @@
_worker_names = {
"code-lint": ["localhost"],
"linux-x86_64-code": ["localhost"],
"linux-x86_64-code-gpu": ["localhost"],
"linux-x86_64-doc-api": ["localhost"],
"linux-x86_64-doc-studio-pipeline": ["localhost"],
"linux-x86_64-general": ["localhost"],
"linux-x86_64-store-snap": ["localhost"],
"linux-x86_64-store-steam": ["localhost"],
"darwin-arm64-code": ["localhost"],
"darwin-arm64-code-gpu": ["localhost"],
"darwin-x86_64-code": ["localhost"],
"darwin-x86_64-code-gpu": ["localhost"],
"windows-amd64-code": ["localhost"],
"windows-amd64-code-gpu": [],
"windows-amd64-store-windows": ["localhost"],
}

53
buildbot/pipeline/code.py
View File

@ -2,7 +2,9 @@
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
# <pep8 compliant>
from functools import partial
import pathlib
import random
import buildbot.plugins
@ -133,6 +135,20 @@ scheduler_properties_common = [
strict=True,
default=False,
),
buildbot.plugins.util.BooleanParameter(
name="needs_gpu_binaries",
label="GPU binaries -> build Cycles GPU kernels",
required=True,
strict=True,
default=False,
),
buildbot.plugins.util.BooleanParameter(
name="needs_gpu_tests",
label="GPU tests -> run EEVEE, Viewport and Cycles GPU tests",
required=True,
strict=True,
default=False,
),
]
# code-daily
@ -209,6 +225,10 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ
patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
override_branch_id = pipeline.common.fetch_property(props, key="override_branch_id", default="")
python_module = pipeline.common.fetch_property(props, key="python_module", default=False)
needs_gpu_tests = pipeline.common.fetch_property(props, key="needs_gpu_tests", default=False)
needs_gpu_binaries = pipeline.common.fetch_property(
props, key="needs_gpu_binaries", default=False
)
build_configuration = pipeline.common.fetch_property(
props, key="build_configuration", default="release"
)
@ -246,8 +266,11 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ
args += ["--python-module"]
if needs_full_clean:
args += ["--needs-full-clean"]
if step_name in ["compile-install"] and needs_package_delivery:
args += ["--needs-gpu-binaries"]
if step_name in ["compile-gpu", "compile-install", "test-code"]:
if needs_package_delivery or needs_gpu_binaries:
args += ["--needs-gpu-binaries"]
if needs_gpu_tests:
args += ["--needs-gpu-tests"]
args += [step_name]
@ -380,6 +403,25 @@ def create_deliver_test_results_step(worker_config, track_id, pipeline_type):
)
def next_worker_code(worker_names_gpu, builder, workers, request):
# Use a GPU worker if needed and supported for this platform.
# GPU worker is currently reserved for GPU builds only.
compatible_workers = []
if request.properties.getProperty("needs_gpu_tests", False) and worker_names_gpu:
for worker in workers:
if worker.worker.workername in worker_names_gpu:
compatible_workers.append(worker)
else:
for worker in workers:
if worker.worker.workername not in worker_names_gpu:
compatible_workers.append(worker)
if not compatible_workers:
return None
return random.choice(compatible_workers)
def populate(devops_env_id):
builders = []
schedulers = []
@ -478,8 +520,10 @@ def populate(devops_env_id):
print(f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders")
worker_group_id = f"{platform_architecture}-code"
worker_group_id_gpu = f"{platform_architecture}-code-gpu"
pipeline_worker_names = platform_worker_names[worker_group_id]
pipeline_worker_names_gpu = platform_worker_names[worker_group_id_gpu]
if pipeline_worker_names:
# Only create the builders if the worker exists
pipeline_builder_name = (
@ -502,6 +546,7 @@ def populate(devops_env_id):
buildbot.plugins.util.BuilderConfig(
name=pipeline_builder_name,
workernames=suitable_pipeline_worker_names,
nextWorker=partial(next_worker_code, pipeline_worker_names_gpu),
tags=pipeline_builder_tags,
factory=pipeline_build_factory,
)
@ -554,6 +599,8 @@ def populate(devops_env_id):
"needs_package_delivery": buildbot.plugins.util.Property(
"needs_package_delivery"
),
"needs_gpu_binaries": buildbot.plugins.util.Property("needs_gpu_binaries"),
"needs_gpu_tests": buildbot.plugins.util.Property("needs_gpu_tests"),
"needs_skip_tests": buildbot.plugins.util.Property("needs_skip_tests"),
"platform_architectures": buildbot.plugins.util.Property(
"platform_architectures"
@ -637,6 +684,7 @@ def populate(devops_env_id):
"python_module": False,
"needs_skip_tests": False,
"needs_package_delivery": False,
"needs_gpu_binaries": False,
"build_configuration": "release",
"platform_architectures": code_tracked_platform_architectures[track_id],
}
@ -663,6 +711,7 @@ def populate(devops_env_id):
"python_module": False,
"needs_skip_tests": False,
"needs_package_delivery": True,
"needs_gpu_binaries": True,
"build_configuration": "release",
"platform_architectures": code_tracked_platform_architectures[track_id],
}

2
buildbot/worker/blender/__init__.py
View File

@ -20,6 +20,7 @@ class CodeBuilder(worker.utils.Builder):
super().__init__(args, "blender", "blender")
self.needs_full_clean = args.needs_full_clean
self.needs_gpu_binaries = args.needs_gpu_binaries
self.needs_gpu_tests = args.needs_gpu_tests
self.needs_ninja = True
self.python_module = args.python_module
self.build_configuration = args.build_configuration
@ -161,6 +162,7 @@ def create_argument_parser(steps: worker.utils.BuilderSteps) -> argparse.Argumen
parser = worker.utils.create_argument_parser(steps=steps)
parser.add_argument("--needs-full-clean", action="store_true", required=False)
parser.add_argument("--needs-gpu-binaries", action="store_true", required=False)
parser.add_argument("--needs-gpu-tests", action="store_true", required=False)
parser.add_argument("--python-module", action="store_true", required=False)
parser.add_argument(
"--build-configuration",

12
buildbot/worker/blender/compile.py
View File

@ -224,9 +224,21 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe
else:
options += [f"-DWITH_CYCLES_ONEAPI_BINARIES=OFF"]
options += [f"-DHIP_ROOT_DIR:PATH=/opt/rocm/hip"]
# GPU render tests support Linux + NVIDIA currently
if builder.needs_gpu_tests:
# TODO fails because missing vulkan headers?
# options += [f"-DWITH_GPU_DRAW_TESTS=ON"]
options += [f"-DWITH_GPU_RENDER_TESTS=ON"]
options += [f"-DWITH_GPU_RENDER_TESTS_SILENT=OFF"]
if builder.needs_gpu_binaries:
options += [f"-DCYCLES_TEST_DEVICES=CPU;OPTIX"]
elif builder.platform == "darwin":
# Metal on macOS
if builder.architecture == "arm64":
if builder.needs_gpu_tests:
options += [f"-DWITH_GPU_RENDER_TESTS=ON"]
options += [f"-DWITH_GPU_RENDER_TESTS_SILENT=OFF"]
options += [f"-DCYCLES_TEST_DEVICES=CPU;METAL"]
if "optix" in buildbotConfig:

12
buildbot/worker/blender/test.py
View File

@ -18,8 +18,10 @@ def get_ctest_arguments(builder: worker.blender.CodeBuilder) -> List[str]:
# Run tests in single threads for tracks which has older OpenEXR/OpenImageIO,
# This prevents idiff.exe dead-lock on exit.
if builder.platform != "windows" or builder.track_id != "v330":
args += ["--parallel", "4"]
if not (builder.platform == "windows" and builder.track_id == "v330"):
# GPU tests are currently slow and can cause timeouts.
if not builder.needs_gpu_tests:
args += ["--parallel", "4"]
args += ["-C", worker.blender.compile.get_cmake_build_type(builder)]
return args
@ -41,8 +43,10 @@ def package_for_upload(builder: worker.blender.CodeBuilder, success: bool) -> No
shutil.make_archive(str(package_filepath), "zip", package_tests_dir, package_filename)
shutil.rmtree(package_filepath)
# Always upload unpacked folder for main and release tracks.
if builder.track_id != "vexp":
# Always upload unpacked folder for main and release tracks,
# when using GPU tests. This is useful for debugging GPU
# differences.
if builder.track_id != "vexp" and builder.needs_gpu_tests:
branch = builder.branch_id.replace("blender-", "").replace("-release", "")
name = f"{branch}-{builder.platform}-{builder.architecture}"
shutil.copytree(build_tests_dir, package_tests_dir / name)