5 Commits

4 changed files with 225 additions and 374 deletions

View File

@@ -1,8 +0,0 @@
{
"project_id" : "Infrastructure: Blender Buildbot",
"conduit_uri" : "https://developer.blender.org/",
"phabricator.uri" : "https://developer.blender.org/",
"git.default-relative-commit" : "origin/master",
"arc.land.update.default" : "rebase",
"arc.land.onto.default" : "master"
}

View File

@@ -1,86 +1,40 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# -*- python -*-
# ex: set syntax=python:
# <pep8 compliant>
from buildbot.www.authz.roles import RolesFromBase
from buildbot.www.authz.roles import RolesFromOwner
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.steps.shell import Test
from buildbot.steps.shell import Compile
from buildbot.steps.shell import ShellCommand
from buildbot.process.properties import Interpolate
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, util
from buildbot.config import BuilderConfig
from buildbot.schedulers import timed, forcesched
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.filter import ChangeFilter
from buildbot.changes.gitpoller import GitPoller
from buildbot.changes.svnpoller import SVNPoller
from buildbot.worker import Worker
from datetime import timedelta
# NOTE: We load the workers and their passwords from a separator file, so we can
# have # this one in Git.
import master_private
# List of the branches being built automatically overnight
NIGHT_SCHEDULE_BRANCHES = ["master"]
# Dictionary that the buildmaster pays attention to.
c = BuildmasterConfig = {}
# Project identity.
c['projectName'] = 'Blender'
c['projectURL'] = 'https://www.blender.org/'
c['title'] = 'Blender'
c['titleURL'] = 'https://builder.blender.org/'
# Buildbot information.
c['buildbotURL'] = 'https://builder.blender.org/admin/'
c['buildbotNetUsageData'] = 'basic'
# Various.
c['db_url'] = 'sqlite:///state.sqlite'
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
c['buildbotNetUsageData'] = None
################################################################################
# BUILD WORKERS
#
# We load the slaves and their passwords from a separator file, so we can have
# this one in SVN.
from buildbot.worker import Worker
import master_private
c['workers'] = []
for worker in master_private.workers:
c['workers'].append(
Worker(worker['name'], worker['password'], max_builds=1))
for slave in master_private.slaves:
c['workers'].append(Worker(slave['name'], slave['password']))
# TCP port through which workers connect
# TCP port through which slaves connect
c['protocols'] = {
'pb': {
'port': 'tcp:{}'.format(9989)
"pb": {
"port": "tcp:{}".format(9989)
}
}
# CHANGE SOURCES
from buildbot.changes.svnpoller import SVNPoller
from buildbot.changes.gitpoller import GitPoller
c['change_source'] = GitPoller('git://git.blender.org/blender.git',
pollinterval=1200)
################################################################################
# CODEBASES
#
# Allow to control separately things like branches for each repo and submodules.
@@ -98,130 +52,111 @@ def codebaseGenerator(chdict):
c['codebaseGenerator'] = codebaseGenerator
################################################################################
# SCHEDULERS
#
# Decide how to react to incoming changes.
from buildbot.schedulers import timed, forcesched
c['schedulers'] = []
def schedule_force_build(name, branch):
def schedule_force_build(name):
"""
Makes it possible to have "Force Build" for the given builder.
Makes sure only reasonable subset of properties are exposed.
Makes sure only reasonabel subset of properties are exposed.
"""
if branch != '':
branch_parameter = forcesched.FixedParameter(
name='branch',
default=branch,
hide=True)
else:
branch_parameter = forcesched.StringParameter(
name='branch',
label='Branch:',
default='custom-branch-name-here',
regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$')
c['schedulers'].append(forcesched.ForceScheduler(
name='force_' + name,
buttonName='Force Build',
buttonName="Force Build",
builderNames=[name],
codebases=[forcesched.CodebaseParameter(
codebase='blender',
branch=branch_parameter,
# Hide revision. We don't want to allow anyone to overwrite the
# master build with an older version. Could be added back once we
# have authentication.
revision=forcesched.FixedParameter(
name='revision',
default='',
hide=True),
repository=forcesched.FixedParameter(
name='repository',
default='',
hide=True),
project=forcesched.FixedParameter(
name='project',
default='',
hide=True)),
]))
codebase="blender",
branch=forcesched.StringParameter(
name="branch",
label="Branch:",
default="master"),
# Hide revision. We don't want to allow anyone to overwrite the
# master build with an older version. Could be added back once we
# have authentication.
revision=forcesched.FixedParameter(
name="revision",
default="",
hide=True),
repository=forcesched.FixedParameter(
name="repository",
default="",
hide=True),
project=forcesched.FixedParameter(
name="project",
default="",
hide=True)),
],
properties=[]))
def schedule_nightly_build(name, branch, hour, minute=0):
def schedule_nightly_build(name, hour, minute=0):
"""
Creates scheduler for nightly builds for a given builder.
"""
scheduler_name = f'nightly_{name} {branch}'
c['schedulers'].append(timed.Nightly(
name=scheduler_name,
codebases={
'blender': {'repository': '',
'branch': branch}},
branch=branch,
builderNames=[name],
hour=hour,
minute=minute))
for current_branch in NIGHT_SCHEDULE_BRANCHES:
scheduler_name = "nightly_" + name
if current_branch:
scheduler_name += ' ' + current_branch
c['schedulers'].append(timed.Nightly(
name=scheduler_name,
codebases={
"blender": {"repository": "",
"branch": current_branch}},
branch=current_branch,
builderNames=[name],
hour=hour,
minute=minute))
def schedule_change_build(name, branch):
"""
Creates scheduler for building on changes.
This will not package and upload the build.
"""
scheduler_name = f'change_{name} {branch}'
c['schedulers'].append(SingleBranchScheduler(
name=scheduler_name,
codebases={
'blender': {'repository': '',
'branch': branch}},
builderNames=[name],
treeStableTimer=120,
change_filter=ChangeFilter(project=['blender'], branch=branch),
properties={'skip_upload': True, 'skip_codesign': True}))
################################################################################
# BUILDERS
#
# The 'builders' list defines the Builders, which tell Buildbot how to
# perform a build: what steps, and which workers can execute them.
# Note that any particular build will only take place on one worker.
# perform a build: what steps, and which slaves can execute them.
# Note that any particular build will only take place on one slave.
from buildbot.config import BuilderConfig
from buildbot.plugins import steps, util
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shell import Compile
from buildbot.steps.shell import Test
from buildbot.steps.transfer import FileUpload
from buildbot.steps.master import MasterShellCommand
# add builder utility
c['builders'] = []
builders_all_branches = set()
buildernames = []
# Add builder utility.
def add_builder(c, name, factory, branch='',
rsync=False, hour=3, minute=0):
workernames = []
def add_builder(c, name, platforms, factory, branch='', hour=3, minute=0, use_nightly_builder=True):
if branch != '':
builders_all_branches.add(branch)
for slave in master_private.slaves:
if name in slave['builders']:
workernames.append(slave['name'])
for platform in platforms:
workernames = []
builder_name = f'{name}_{platform}'
if workernames:
f = factory(name, branch, rsync)
c['builders'].append(BuilderConfig(name=name,
workernames=workernames,
factory=f,
tags=['blender']))
buildernames.append(name)
for worker in master_private.workers:
if platform == worker['platform']:
workernames.append(worker['name'])
builder_name = f"{worker['platform_short']}_{name}"
if workernames:
f = factory(builder_name, branch)
c['builders'].append(BuilderConfig(name=builder_name,
workernames=workernames,
factory=f,
tags=['blender']))
if use_nightly_builder:
schedule_nightly_build(builder_name, branch, hour, minute)
schedule_change_build(builder_name, branch)
schedule_force_build(builder_name, branch)
schedule_nightly_build(name, hour, minute)
schedule_force_build(name)
# Common steps.
# common steps
def git_step(branch=''):
if branch:
@@ -241,118 +176,79 @@ def git_step(branch=''):
submodules=True)
# Generic builder.
def rsync_step(python_command, id, branch, rsync_script):
return ShellCommand(name='rsync',
command=[python_command, rsync_script, id, branch],
description='uploading',
descriptionDone='uploaded',
workdir='install')
def do_upload(step):
return not step.hasProperty('skip_upload')
@util.renderer
def script_command(props, script, id, branch):
# NOTE: On Windows never includes major version in the executable name,
# so Python 3 will have be 'python.exe'.
# generic builder
def generic_builder(id, branch='', rsync=False):
filename = 'uploaded/buildbot_upload_' + id + '.zip'
update_script = '../blender.git/build_files/buildbot/slave_update.py'
compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
test_script = '../blender.git/build_files/buildbot/slave_test.py'
pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
unpack_script = 'master_unpack.py'
f = BuildFactory()
if id.startswith('win'):
python_command = 'python'
else:
python_command = 'python3'
git_branch = branch or Interpolate('%(src:blender:branch)s')
args = [python_command, script, id, git_branch]
if not props.hasProperty('skip_codesign'):
args += ['--codesign']
return args
def generic_builder(id, branch=''):
# TODO(sergey): Consider using pathlib.
filename = f'uploaded/buildbot_upload_{id}.zip'
update_script = '../blender.git/build_files/buildbot/worker_update.py'
compile_script = '../blender.git/build_files/buildbot/worker_compile.py'
test_script = '../blender.git/build_files/buildbot/worker_test.py'
pack_script = '../blender.git/build_files/buildbot/worker_pack.py'
unpack_script = 'master_unpack.py'
f = BuildFactory()
f.addStep(git_step(branch))
git_branch = branch or Interpolate('%(src:blender:branch)s')
f.addStep(ShellCommand(
name='submodules and libraries update',
command=script_command.withArgs(update_script, id, branch),
command=[python_command, update_script, id, git_branch],
description='updating',
descriptionDone='updated',
haltOnFailure=True))
descriptionDone='updated'))
f.addStep(Compile(
command=script_command.withArgs(compile_script, id, branch),
command=[python_command, compile_script, id, git_branch],
timeout=3600))
f.addStep(Test(
command=script_command.withArgs(test_script, id, branch)))
command=[python_command, test_script, id, git_branch]))
f.addStep(ShellCommand(
name='package',
command=script_command.withArgs(pack_script, id, branch),
command=[python_command, pack_script, id, git_branch],
description='packaging',
descriptionDone='packaged',
haltOnFailure=True,
doStepIf=do_upload))
descriptionDone='packaged'))
f.addStep(FileUpload(name='upload',
workersrc='buildbot_upload.zip',
masterdest=filename,
maxsize=500 * 1024 * 1024,
workdir='install',
doStepIf=do_upload))
if rsync:
f.addStep(rsync_step(python_command, id, git_branch, rsync_script))
else:
f.addStep(FileUpload(name='upload',
workersrc='buildbot_upload.zip',
masterdest=filename,
maxsize=180 * 1024 * 1024,
workdir='install'))
f.addStep(MasterShellCommand(name='unpack',
command=['python3.7',
command=['python3.6',
unpack_script,
filename],
description='unpacking',
descriptionDone='unpacked',
doStepIf=do_upload))
descriptionDone='unpacked'))
return f
################################################################################
# Builders
add_builder(c,
'master',
['windows', 'macOS_10_15', 'linux_centos7'],
generic_builder,
branch='master',
hour=1,
use_nightly_builder=True)
add_builder(c,
'292',
['windows', 'macOS_10_15', 'linux_centos7'],
generic_builder,
branch='blender-v2.92-release',
hour=1,
use_nightly_builder=False)
add_builder(c,
'lts_283',
['windows', 'macOS_10_13', 'linux_centos7'],
generic_builder,
branch='blender-v2.83-release',
hour=1,
use_nightly_builder=False)
add_builder(c,
'custom_branch',
['windows', 'macOS_10_15', 'linux_centos7'],
generic_builder,
branch='',
hour=1,
use_nightly_builder=False)
add_builder(c, 'mac_x86_64_10_9_cmake', generic_builder, hour=1)
add_builder(c, 'linux_glibc217_x86_64_cmake', generic_builder, hour=1)
# NOTE: Visual Studio 2017 (vc15) is using libraries folder from
# Visual Studio 2015 (vc14)
add_builder(c, 'win64_cmake_vs2017', generic_builder, hour=1)
################################################################################
# CHANGE SOURCES
c['change_source'] = GitPoller(repourl='git://git.blender.org/blender.git',
pollinterval=120,
project='blender',
branches=list(builders_all_branches))
################################################################################
# HORIZONS
from datetime import timedelta
c['changeHorizon'] = 300
@@ -364,17 +260,15 @@ c['configurators'] = [util.JanitorConfigurator(
dayOfWeek=6)]
################################################################################
# WWW
c['www'] = dict(port=8010,
plugins={'console_view': {},
'grid_view': {},
'waterfall_view': {}})
################################################################################
# Access
from buildbot.www.authz.roles import RolesFromOwner
from buildbot.www.authz.roles import RolesFromBase
class StrangerRoles(RolesFromBase):
def getRolesFromUser(self, userDetails):
@@ -382,31 +276,42 @@ class StrangerRoles(RolesFromBase):
authz = util.Authz(
# Simple matcher with '*' glob character.
# Could be util.reStrMatcher if regular expressions are preferred.
stringsMatcher=util.fnmatchStrMatcher,
stringsMatcher=util.fnmatchStrMatcher, # simple matcher with '*' glob character
# stringsMatcher = util.reStrMatcher, # if you prefer regular expressions
allowRules=[
# Admins can do anything,
#
# defaultDeny=False: if user does not have the admin role, we continue
# parsing rules
util.AnyEndpointMatcher(role='admins', defaultDeny=False),
# admins can do anything,
# defaultDeny=False: if user does not have the admin role, we continue parsing rules
util.AnyEndpointMatcher(role="admins", defaultDeny=False),
# Defaulting old config, everyone can stop build.
util.StopBuildEndpointMatcher(role='stranger'),
util.ForceBuildEndpointMatcher(role='stranger'),
util.RebuildBuildEndpointMatcher(role='stranger'),
util.EnableSchedulerEndpointMatcher(role='admins'),
util.StopBuildEndpointMatcher(role="stranger"),
util.ForceBuildEndpointMatcher(role="stranger"),
util.RebuildBuildEndpointMatcher(role="stranger"),
util.EnableSchedulerEndpointMatcher(role="admins"),
# If future Buildbot implement new control, we are safe with this last
# rule.
#
# if future Buildbot implement new control, we are safe with this last rule
# NOTE: This prevents us from cancelling queue, so disabled for now.
# util.AnyControlEndpointMatcher(role='admins')
# util.AnyControlEndpointMatcher(role="admins")
],
roleMatchers=[
RolesFromOwner(role='owner'),
RolesFromOwner(role="owner"),
StrangerRoles(),
])
c['www']['authz'] = authz
# PROJECT IDENTITY
c['projectName'] = "Blender"
c['projectURL'] = "https://www.blender.org"
# Buildbot information
c['buildbotURL'] = "https://builder.blender.org/admin/"
c['buildbotNetUsageData'] = 'basic'
# Various
c['db_url'] = "sqlite:///state.sqlite"
c['title'] = "Blender"
c['titleURL'] = "https://builder.blender.org/"
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
c['buildbotNetUsageData'] = None

View File

@@ -1,18 +1,5 @@
workers = [
{'name': 'linux_something_something',
'password': 'something',
'platform': 'linux_centos7',
'platform_short': 'linux'},
{'name': 'macOS_10_15_something_something',
'password': 'something',
'platform': 'macOS_10_15',
'platform_short': 'macos'},
{'name': 'macOS_10_13_something_something',
'password': 'something',
'platform': 'macOS_10_13',
'platform_short': 'macos'},
{'name': 'windows_something_something',
'password': 'something',
'platform': 'windows',
'platform_short': 'windows'}
slaves = [
{'name': 'linux_glibc219_x86_64_chroot',
'password': 'barbarianna',
'builders': ['linux_glibc219_x86_64_cmake']}
]

View File

@@ -25,83 +25,79 @@ import argparse
import os
import shutil
import sys
import time
import zipfile
DOWNLOAD_DIR = "/data/www/vhosts/builder.blender.org/webroot/download/"
# Parse package filename to extract branch and platform
class Package:
def __init__(self, zipname):
self.zipname = zipname
self.filename = os.path.basename(zipname)
self.version = ""
self.branch = ""
self.platform = ""
self.parsed = False
self.parse_filename()
self.platform = self._get_platform(self.filename)
self.branch = self._get_branch(self.filename)
# extension stripping
def _strip_extension(self, filename):
filename, _ = os.path.splitext(filename)
if filename.endswith('.tar'):
filename, _ = os.path.splitext(filename)
extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
for ext in extensions:
if filename.endswith(ext):
filename = filename[:-len(ext)]
return filename
# extract platform from package name
def parse_filename(self):
# Name is one one of:
# blender-version-platform: official release
# blender-version-hash-platform: daily build
# branch-blender-version-hash-platform: branch build
filename = self._strip_extension(self.filename)
def _get_platform(self, filename):
# name is blender-version-platform.extension. we want to get the
# platform out, but there may be some variations, so we fiddle a
# bit to handle current and hopefully future names
filename = self._strip_extension(filename)
filename = self._strip_extension(filename)
tokens = filename.split("-")
if len(tokens) < 3:
return
if "blender" not in tokens:
return
blender_index = tokens.index("blender")
if not (blender_index + 2 < len(tokens)):
return
# Detect branch.
branch = "-".join(tokens[:blender_index])
# Detect version.
if branch == "":
version = tokens[blender_index + 1]
if len(tokens) == 3:
version += " release"
else:
version = ""
# Detect platform.
platform = tokens[-1]
platforms = ('osx', 'mac', 'bsd',
'win', 'linux', 'source',
'irix', 'solaris', 'mingw')
platform_found = False
for name in platforms:
if platform.lower().startswith(name):
platform_found = True
if not platform_found:
return
platform_tokens = []
found = False
for i, token in enumerate(tokens):
if not found:
for platform in platforms:
if platform in token.lower():
found = True
break
if found:
platform_tokens += [token]
return '-'.join(platform_tokens)
def _get_branch(self, filename):
tokens = filename.split("-")
branch = ""
for token in tokens:
if token == "blender":
return branch
if branch == "":
branch = token
else:
branch = branch + "-" + token
return ""
self.version = version
self.platform = platform
self.branch = branch
self.parsed = True
def get_download_dir(branch):
download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
if not branch or branch == 'master':
directory = DOWNLOAD_DIR
directory = download_prefix
elif branch == 'experimental-build':
directory = os.path.join(download_prefix, "experimental")
else:
directory = os.path.join(DOWNLOAD_DIR, branch)
directory = os.path.join(download_prefix, branch)
os.makedirs(directory, exist_ok=True)
os.chmod(directory, 0o755)
return directory
@@ -120,10 +116,6 @@ def open_zipfile(filename):
def get_zipfile_packages(zipfile):
# List packages in zip file
packages = [Package(zipname) for zipname in zipfile.namelist()]
for package in packages:
if not package.parsed:
sys.stderr.write("Failed to parse package filename: %s\n" % str(package.filename))
sys.exit(1)
if len(packages) == 0:
sys.stderr.write('Empty zip file\n')
sys.exit(1)
@@ -133,7 +125,6 @@ def get_branch_platform(packages):
# Extract branch and platform names
branch = packages[0].branch
platform = packages[0].platform
version = packages[0].version
if platform == '':
sys.stderr.write('Failed to detect platform ' +
@@ -141,11 +132,11 @@ def get_branch_platform(packages):
sys.exit(1)
for package in packages:
if package.branch != branch or package.platform != platform or package.version != version:
if package.branch != branch or package.platform != platform:
sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
sys.exit(1)
return branch, platform, version
return branch, platform
def extract_zipfile_packages(zipfile, packages, branch):
# Extract packages from zip file
@@ -166,14 +157,13 @@ def extract_zipfile_packages(zipfile, packages, branch):
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
sys.exit(1)
def remove_replaced_packages(branch, platform, version, new_packages):
# Remove other files from the same platform and branch that are replaced
# by the new packages.
def remove_older_packages(branch, platform, new_packages):
# Remove other files from the same platform and branch
directory = get_download_dir(branch)
for filename in os.listdir(directory):
package = Package(filename)
if package.platform == platform and package.branch == branch and package.version == version:
if package.platform == platform and package.branch == branch:
is_new_package = False
for new_package in new_packages:
if package.filename == new_package.filename:
@@ -181,32 +171,10 @@ def remove_replaced_packages(branch, platform, version, new_packages):
if not is_new_package:
try:
print("Removing older package version", filename)
os.remove(os.path.join(directory, filename))
except Exception as ex:
sys.stderr.write('Failed to remove replaced package: %s\n' % str(ex))
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
def remove_old_packages():
# Remove any branch packages that are 100 days or older.
cutoff_time = time.time() - 100 * 86400
for dirname in os.listdir(DOWNLOAD_DIR):
dirpath = os.path.join(DOWNLOAD_DIR, dirname)
if not os.path.isdir(dirpath):
continue
for filename in os.listdir(dirpath):
filepath = os.path.join(dirpath, filename)
if not os.path.isfile(filepath):
continue
file_mtime = os.stat(filepath).st_mtime
if file_mtime < cutoff_time:
try:
print("Removing old branch build", filename)
os.remove(filepath)
except Exception as ex:
sys.stderr.write('Failed to remove old package: %s\n' % str(ex))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
@@ -215,8 +183,7 @@ if __name__ == "__main__":
with open_zipfile(args.filename) as zipfile:
packages = get_zipfile_packages(zipfile)
branch, platform, version = get_branch_platform(packages)
branch, platform = get_branch_platform(packages)
extract_zipfile_packages(zipfile, packages, branch)
remove_replaced_packages(branch, platform, version, packages)
remove_old_packages()
remove_older_packages(branch, platform, packages)