buildbot/conf/local/machines.py

@@ -5,14 +5,18 @@
 _worker_names = {
     "code-lint": ["localhost"],
     "linux-x86_64-code": ["localhost"],
+    "linux-x86_64-code-gpu": ["localhost"],
     "linux-x86_64-doc-api": ["localhost"],
     "linux-x86_64-doc-studio-pipeline": ["localhost"],
     "linux-x86_64-general": ["localhost"],
     "linux-x86_64-store-snap": ["localhost"],
     "linux-x86_64-store-steam": ["localhost"],
     "darwin-arm64-code": ["localhost"],
+    "darwin-arm64-code-gpu": ["localhost"],
     "darwin-x86_64-code": ["localhost"],
+    "darwin-x86_64-code-gpu": ["localhost"],
     "windows-amd64-code": ["localhost"],
+    "windows-amd64-code-gpu": [],
     "windows-amd64-store-windows": ["localhost"],
 }
 

buildbot/pipeline/code.py

@@ -2,7 +2,9 @@
 # SPDX-FileCopyrightText: 2011-2024 Blender Authors
 # <pep8 compliant>
 
+from functools import partial
 import pathlib
+import random
 
 import buildbot.plugins
 
@@ -133,6 +135,20 @@ scheduler_properties_common = [
         strict=True,
         default=False,
     ),
+    buildbot.plugins.util.BooleanParameter(
+        name="needs_gpu_binaries",
+        label="GPU binaries -> build Cycles GPU kernels",
+        required=True,
+        strict=True,
+        default=False,
+    ),
+    buildbot.plugins.util.BooleanParameter(
+        name="needs_gpu_tests",
+        label="GPU tests -> run EEVEE, Viewport and Cycles GPU tests",
+        required=True,
+        strict=True,
+        default=False,
+    ),
 ]
 
 # code-daily
@@ -209,6 +225,10 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ
     patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
     override_branch_id = pipeline.common.fetch_property(props, key="override_branch_id", default="")
     python_module = pipeline.common.fetch_property(props, key="python_module", default=False)
+    needs_gpu_tests = pipeline.common.fetch_property(props, key="needs_gpu_tests", default=False)
+    needs_gpu_binaries = pipeline.common.fetch_property(
+        props, key="needs_gpu_binaries", default=False
+    )
     build_configuration = pipeline.common.fetch_property(
         props, key="build_configuration", default="release"
     )
@@ -246,8 +266,11 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ
         args += ["--python-module"]
     if needs_full_clean:
         args += ["--needs-full-clean"]
-    if step_name in ["compile-install"] and needs_package_delivery:
+    if step_name in ["compile-gpu", "compile-install"]:
+        if needs_package_delivery or needs_gpu_binaries:
             args += ["--needs-gpu-binaries"]
+        if needs_gpu_tests:
+            args += ["--needs-gpu-tests"]
 
     args += [step_name]
 
@@ -380,6 +403,21 @@ def create_deliver_test_results_step(worker_config, track_id, pipeline_type):
     )
 
 
+def next_worker_code(worker_names_gpu, builder, workers, request):
+    # Use a GPU worker if needed and supported for this platform.
+    if not workers:
+        return None
+
+    if not request.properties.getProperty("needs_gpu_tests", False) or not worker_names_gpu:
+        return random.choice(workers)
+
+    for worker in workers:
+        if worker.worker.workername in worker_names_gpu:
+            return worker
+
+    return None
+
+
 def populate(devops_env_id):
     builders = []
     schedulers = []
@@ -478,8 +516,10 @@ def populate(devops_env_id):
                 print(f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders")
 
                 worker_group_id = f"{platform_architecture}-code"
+                worker_group_id_gpu = f"{platform_architecture}-code-gpu"
 
                 pipeline_worker_names = platform_worker_names[worker_group_id]
+                pipeline_worker_names_gpu = platform_worker_names[worker_group_id_gpu]
                 if pipeline_worker_names:
                     # Only create the builders if the worker exists
                     pipeline_builder_name = (
@@ -502,6 +542,7 @@ def populate(devops_env_id):
                         buildbot.plugins.util.BuilderConfig(
                             name=pipeline_builder_name,
                             workernames=suitable_pipeline_worker_names,
+                            nextWorker=partial(next_worker_code, pipeline_worker_names_gpu),
                             tags=pipeline_builder_tags,
                             factory=pipeline_build_factory,
                         )
@@ -554,6 +595,8 @@ def populate(devops_env_id):
                     "needs_package_delivery": buildbot.plugins.util.Property(
                         "needs_package_delivery"
                     ),
+                    "needs_gpu_binaries": buildbot.plugins.util.Property("needs_gpu_binaries"),
+                    "needs_gpu_tests": buildbot.plugins.util.Property("needs_gpu_tests"),
                     "needs_skip_tests": buildbot.plugins.util.Property("needs_skip_tests"),
                     "platform_architectures": buildbot.plugins.util.Property(
                         "platform_architectures"
@@ -637,6 +680,7 @@ def populate(devops_env_id):
                             "python_module": False,
                             "needs_skip_tests": False,
                             "needs_package_delivery": False,
+                            "needs_gpu_binaries": False,
                             "build_configuration": "release",
                             "platform_architectures": code_tracked_platform_architectures[track_id],
                         }
@@ -663,6 +707,7 @@ def populate(devops_env_id):
                             "python_module": False,
                             "needs_skip_tests": False,
                             "needs_package_delivery": True,
+                            "needs_gpu_binaries": True,
                             "build_configuration": "release",
                             "platform_architectures": code_tracked_platform_architectures[track_id],
                         }

buildbot/worker/blender/__init__.py

@@ -20,6 +20,7 @@ class CodeBuilder(worker.utils.Builder):
         super().__init__(args, "blender", "blender")
         self.needs_full_clean = args.needs_full_clean
         self.needs_gpu_binaries = args.needs_gpu_binaries
+        self.needs_gpu_tests = args.needs_gpu_tests
         self.needs_ninja = True
         self.python_module = args.python_module
         self.build_configuration = args.build_configuration
@@ -161,6 +162,7 @@ def create_argument_parser(steps: worker.utils.BuilderSteps) -> argparse.Argumen
     parser = worker.utils.create_argument_parser(steps=steps)
     parser.add_argument("--needs-full-clean", action="store_true", required=False)
     parser.add_argument("--needs-gpu-binaries", action="store_true", required=False)
+    parser.add_argument("--needs-gpu-tests", action="store_true", required=False)
     parser.add_argument("--python-module", action="store_true", required=False)
     parser.add_argument(
         "--build-configuration",

buildbot/worker/blender/compile.py

@@ -224,9 +224,21 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe
         else:
             options += [f"-DWITH_CYCLES_ONEAPI_BINARIES=OFF"]
         options += [f"-DHIP_ROOT_DIR:PATH=/opt/rocm/hip"]
+
+        # GPU render tests support Linux + NVIDIA currently
+        if builder.needs_gpu_tests:
+            # TODO fails because missing vulkan headers?
+            # options += [f"-DWITH_GPU_DRAW_TESTS=ON"]
+            options += [f"-DWITH_GPU_RENDER_TESTS=ON"]
+            options += [f"-DWITH_GPU_RENDER_TESTS_SILENT=OFF"]
+            if builder.needs_gpu_binaries:
+                options += [f"-DCYCLES_TEST_DEVICES=CPU;OPTIX"]
     elif builder.platform == "darwin":
         # Metal on macOS
         if builder.architecture == "arm64":
+            if builder.needs_gpu_tests:
+                options += [f"-DWITH_GPU_RENDER_TESTS=ON"]
+                options += [f"-DWITH_GPU_RENDER_TESTS_SILENT=OFF"]
             options += [f"-DCYCLES_TEST_DEVICES=CPU;METAL"]
 
     if "optix" in buildbotConfig:

buildbot/worker/blender/test.py

@@ -18,7 +18,9 @@ def get_ctest_arguments(builder: worker.blender.CodeBuilder) -> List[str]:
 
     # Run tests in single threads for tracks which has older OpenEXR/OpenImageIO,
     # This prevents idiff.exe dead-lock on exit.
-    if builder.platform != "windows" or builder.track_id != "v330":
+    if not (builder.platform == "windows" and builder.track_id == "v330"):
+        # GPU tests are currently slow and can cause timeouts.
+        if not builder.needs_gpu_tests:
             args += ["--parallel", "4"]
 
     args += ["-C", worker.blender.compile.get_cmake_build_type(builder)]
@@ -53,8 +55,14 @@ def test(builder: worker.blender.CodeBuilder) -> None:
     os.chdir(builder.build_dir)
     success = False
 
+    env = os.environ.copy()
+
+    # Needed to successfully initialize EGL on Rocky Linux machine.
+    if builder.needs_gpu_tests and builder.platform == "linux":
+        env["DISPLAY"] = ":0"
+
     try:
-        builder.call(["ctest"] + get_ctest_arguments(builder))
+        builder.call(["ctest"] + get_ctest_arguments(builder), env=env)
         success = True
     finally:
         package_for_upload(builder, success)