infrastructure
/
blender-devops
1
1
Fork 2
Code Pull Requests Activity

Windows: Initial ARM64 Support #3

Manually merged
Brecht Van Lommel merged 2 commits from windows-arm64 into main 2024-03-29 15:21:16 +01:00
Conversation 1 Commits 2 Files Changed 7 +97 -54
7 changed files with 97 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
buildbot/conf/branches.py
View File

@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
# <pep8 compliant>
import copy
from collections import OrderedDict
# Blender repository branches used for daily builds and API doc generation.
@ -14,7 +16,7 @@ code_tracked_branch_ids = {
}
# Processor architectures to build for each track.
code_tracked_platform_architectures = {
code_official_platform_architectures = {
"vdev": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"],
"vexp": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"],
"v330": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"],
@ -22,6 +24,11 @@ code_tracked_platform_architectures = {
"v410": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"],
}
# Windows ARM64 not used by default yet.
code_all_platform_architectures = copy.deepcopy(code_official_platform_architectures)
code_all_platform_architectures["vdev"].append("windows-arm64")
code_all_platform_architectures["vexp"].append("windows-arm64")
track_major_minor_versions = {
"vdev": "4.2",
"vexp": "4.2",

2
buildbot/conf/local/machines.py
View File

@ -18,6 +18,8 @@ _worker_names = {
"windows-amd64-code": ["localhost"],
"windows-amd64-code-gpu": [],
"windows-amd64-store-windows": ["localhost"],
"windows-arm64-code": ["localhost"],
"windows-arm64-code-gpu": [],
}

37
buildbot/pipeline/code.py
View File

@ -91,9 +91,11 @@ code_python_module_skip_test_names = ["sign-code-binaries"]
# Tracks.
code_tracked_branch_ids = conf.branches.code_tracked_branch_ids
code_track_ids = list(code_tracked_branch_ids.keys())
code_tracked_platform_architectures = conf.branches.code_tracked_platform_architectures
code_all_platform_architectures = conf.branches.code_all_platform_architectures
code_official_platform_architectures = conf.branches.code_official_platform_architectures
code_track_pipeline_types = {}
track_properties = {}
for track, branch in code_tracked_branch_ids.items():
if track == "vdev":
code_track_pipeline_types[track] = ["daily"]
@ -102,17 +104,21 @@ for track, branch in code_tracked_branch_ids.items():
else:
code_track_pipeline_types[track] = ["daily"]
# Properties.
# Track properties.
track_properties[track] = [
buildbot.plugins.util.ChoiceStringParameter(
name="platform_architectures",
label="Platforms:",
required=True,
choices=code_all_platform_architectures[track],
multiple=True,
strict=True,
default=code_official_platform_architectures[track],
),
]
# Scheduler properties.
scheduler_properties_common = [
buildbot.plugins.util.ChoiceStringParameter(
name="platform_architectures",
label="Platforms:",
required=True,
choices=code_tracked_platform_architectures["vdev"],
multiple=True,
strict=True,
default=code_tracked_platform_architectures["vdev"],
),
buildbot.plugins.util.BooleanParameter(
name="python_module",
label="Python module -> build bpy module instead of Blender",
@ -518,7 +524,7 @@ def populate(devops_env_id):
trigger_factory = buildbot.plugins.util.BuildFactory()
# Create builders.
for platform_architecture in code_tracked_platform_architectures[track_id]:
for platform_architecture in code_all_platform_architectures[track_id]:
print(f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders")
worker_group_id = f"{platform_architecture}-code"
@ -670,7 +676,8 @@ def populate(devops_env_id):
hide=True,
)
],
properties=scheduler_properties[f"code-{pipeline_type}"],
properties=track_properties[track_id]
+ scheduler_properties[f"code-{pipeline_type}"],
)
]
@ -688,7 +695,7 @@ def populate(devops_env_id):
"needs_package_delivery": False,
"needs_gpu_binaries": False,
"build_configuration": "release",
"platform_architectures": code_tracked_platform_architectures[track_id],
"platform_architectures": code_all_platform_architectures[track_id],
}
change_filter = buildbot.plugins.util.ChangeFilter(
@ -715,7 +722,7 @@ def populate(devops_env_id):
"needs_package_delivery": True,
"needs_gpu_binaries": True,
"build_configuration": "release",
"platform_architectures": code_tracked_platform_architectures[track_id],
"platform_architectures": code_all_platform_architectures[track_id],
}
nighlty_codebases = {
"blender.git": {

45
buildbot/worker/blender/compile.py
View File

@ -196,25 +196,26 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe
options += [f"-DWITH_GTESTS={with_gtests_state}"]
if builder.platform == "windows":
# CUDA + HIP + oneAPI on Windows
options += [f"-DWITH_CYCLES_CUDA_BINARIES={with_gpu_binaries_state}"]
options += [f"-DWITH_CYCLES_HIP_BINARIES={with_gpu_binaries_state}"]
if can_enable_oneapi_binaries:
options += [f"-DWITH_CYCLES_ONEAPI_BINARIES={with_gpu_binaries_state}"]
options += [f"-DSYCL_OFFLINE_COMPILER_PARALLEL_JOBS=2"]
else:
options += [f"-DWITH_CYCLES_ONEAPI_BINARIES=OFF"]
if "hip" in buildbotConfig:
hip_version = buildbotConfig["hip"]["version"]
else:
hip_version = "5.2.21440"
if "ocloc" in buildbotConfig:
ocloc_version = buildbotConfig["ocloc"]["version"]
else:
ocloc_version = "dev_01"
options += [f"-DHIP_ROOT_DIR=C:/ProgramData/AMD/HIP/hip_sdk_{hip_version}"]
options += [f"-DHIP_PERL_DIR=C:/ProgramData/AMD/HIP/strawberry/perl/bin"]
options += [f"-DOCLOC_INSTALL_DIR=C:/ProgramData/Intel/ocloc/ocloc_{ocloc_version}"]
if builder.architecture != "arm64":
# CUDA + HIP + oneAPI on Windows
options += [f"-DWITH_CYCLES_CUDA_BINARIES={with_gpu_binaries_state}"]
options += [f"-DWITH_CYCLES_HIP_BINARIES={with_gpu_binaries_state}"]
if can_enable_oneapi_binaries:
options += [f"-DWITH_CYCLES_ONEAPI_BINARIES={with_gpu_binaries_state}"]
options += [f"-DSYCL_OFFLINE_COMPILER_PARALLEL_JOBS=2"]
else:
options += [f"-DWITH_CYCLES_ONEAPI_BINARIES=OFF"]
if "hip" in buildbotConfig:
hip_version = buildbotConfig["hip"]["version"]
else:
hip_version = "5.2.21440"
if "ocloc" in buildbotConfig:
ocloc_version = buildbotConfig["ocloc"]["version"]
else:
ocloc_version = "dev_01"
options += [f"-DHIP_ROOT_DIR=C:/ProgramData/AMD/HIP/hip_sdk_{hip_version}"]
options += [f"-DHIP_PERL_DIR=C:/ProgramData/AMD/HIP/strawberry/perl/bin"]
options += [f"-DOCLOC_INSTALL_DIR=C:/ProgramData/Intel/ocloc/ocloc_{ocloc_version}"]
elif builder.platform == "linux":
# CUDA on Linux
options += [f"-DWITH_CYCLES_CUDA_BINARIES={with_gpu_binaries_state}"]
@ -248,7 +249,7 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe
if "optix" in buildbotConfig:
optix_version = buildbotConfig["optix"]["version"]
if builder.platform == "windows":
if builder.platform == "windows" and builder.architecture != "arm64":
options += [
f"-DOPTIX_ROOT_DIR:PATH=C:/ProgramData/NVIDIA Corporation/OptiX SDK {optix_version}"
]
@ -261,7 +262,7 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe
if "hiprt" in buildbotConfig:
hiprt_version = buildbotConfig["hiprt"]["version"]
if builder.platform == "windows":
if builder.platform == "windows" and builder.architecture != "arm64":
options += [
f"-DHIPRT_ROOT_DIR:PATH=C:/ProgramData/AMD/HIP/hiprtsdk-{hiprt_version}/hiprt{hiprt_version}"
]
@ -298,7 +299,7 @@ def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSe
except:
have_cuda12 = False
if builder.platform == "windows":
if builder.platform == "windows" and builder.architecture != "arm64":
# CUDA 10
cuda10_path = pathlib.Path(
f"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v{cuda10_folder_version}"

13
buildbot/worker/deploy/monitor.py
View File

@ -7,6 +7,7 @@ import time
import urllib.request
import worker.blender.version
import worker.deploy.artifacts
import worker.deploy
import worker.utils
@ -34,8 +35,16 @@ def monitor(builder: worker.deploy.CodeDeployBuilder) -> None:
stop_on_required_site_found = False
# 3 windows + 2 mac + 1 linux + 2 checksums
expected_file_count = 3 + 2 + 1 + 2
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
expected_file_count = len(worker.deploy.artifacts.checksums)
for expected_platform in expected_platforms:
if expected_platform.startswith("windows"):
expected_file_count += 3 # msi, msix, zip
else:
expected_file_count += 1
folder_name = f"Blender{version_info.short_version}"
file_pattern = rf"[Bb]lender-{version_info.version}[\.\-\_a-zA-Z0-9]*"

9
buildbot/worker/deploy/pypi.py
View File

@ -57,7 +57,7 @@ def pull(builder: worker.deploy.CodeDeployBuilder) -> None:
# Check expected platforms
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_tracked_platform_architectures[builder.track_id]
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
if len(expected_platforms) != len(matching_builds):
platform_names = "\n".join(expected_platforms)
raise Exception("Unexpected number of builds, expected:\n" + platform_names)
@ -84,14 +84,15 @@ def deliver(builder: worker.deploy.CodeDeployBuilder) -> None:
# Check expected platforms
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_tracked_platform_architectures[builder.track_id]
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
wheel_names = "\n".join([wheel.name for wheel in wheels])
wheel_paths = [str(wheel) for wheel in wheels]
print(wheel_names)
if len(expected_platforms) != len(wheels):
raise Exception("Unexpected number of wheels:\n" + wheel_names)
# Check wheels
cmd = ["twine", "check"] + wheels
cmd = ["twine", "check"] + wheel_paths
worker.utils.call(cmd)
# Upload
@ -101,5 +102,5 @@ def deliver(builder: worker.deploy.CodeDeployBuilder) -> None:
env["TWINE_PASSWORD"] = worker_config.pypi_token(builder.service_env_id)
env["TWINE_REPOSITORY_URL"] = "https://upload.pypi.org/legacy/"
cmd = ["twine", "upload", "--verbose", "--non-interactive"] + wheels
cmd = ["twine", "upload", "--verbose", "--non-interactive"] + wheel_paths
worker.utils.call(cmd, env=env, dry_run=dry_run)

36
buildbot/worker/deploy/windows.py
View File

@ -16,15 +16,9 @@ import worker.deploy
import worker.utils
def package(builder: worker.deploy.CodeStoreBuilder) -> None:
dry_run = False
if not builder.platform == "windows":
if builder.service_env_id == "LOCAL":
worker.utils.warning("Performing dry run on LOCAL service environment")
dry_run = True
else:
raise Exception("Can only run this on Windows, aborting")
def _package_architecture(
builder: worker.deploy.CodeStoreBuilder, architecture: str, dry_run: bool
) -> None:
version_info = worker.blender.version.VersionInfo(builder)
# Revision with MS Store must be set to 0
@ -52,7 +46,11 @@ def package(builder: worker.deploy.CodeStoreBuilder) -> None:
input_file_path = None
for build in builds:
if build["platform"] == "windows" and build["file_extension"] == "zip":
if (
build["platform"] == "windows"
and build["file_extension"] == "zip"
and build["architecture"] == architecture
):
input_file_path = builder.package_dir / build["file_name"]
break
if not input_file_path:
@ -98,5 +96,23 @@ def package(builder: worker.deploy.CodeStoreBuilder) -> None:
worker.blender.pack.generate_file_hash(dest_path)
def package(builder: worker.deploy.CodeStoreBuilder) -> None:
dry_run = False
if not builder.platform == "windows":
if builder.service_env_id == "LOCAL":
worker.utils.warning("Performing dry run on LOCAL service environment")
dry_run = True
else:
raise Exception("Can only run this on Windows, aborting")
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
for expected_platform in expected_platforms:
if expected_platform.startswith("windows"):
architecture = expected_platform.split("-")[1]
_package_architecture(builder, architecture, dry_run)
def deliver(builder: worker.deploy.CodeStoreBuilder) -> None:
worker.utils.info("Windows store delivery not implemented")