413 lines
13 KiB
INI
413 lines
13 KiB
INI
# ##### BEGIN GPL LICENSE BLOCK #####
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software Foundation,
|
|
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
#
|
|
# ##### END GPL LICENSE BLOCK #####
|
|
|
|
# -*- python -*-
|
|
# ex: set syntax=python:
|
|
|
|
# <pep8 compliant>
|
|
|
|
|
|
from buildbot.www.authz.roles import RolesFromBase
|
|
from buildbot.www.authz.roles import RolesFromOwner
|
|
from buildbot.steps.master import MasterShellCommand
|
|
from buildbot.steps.transfer import FileUpload
|
|
from buildbot.steps.shell import Test
|
|
from buildbot.steps.shell import Compile
|
|
from buildbot.steps.shell import ShellCommand
|
|
from buildbot.process.properties import Interpolate
|
|
from buildbot.process.factory import BuildFactory
|
|
from buildbot.plugins import steps, util
|
|
from buildbot.config import BuilderConfig
|
|
from buildbot.schedulers import timed, forcesched
|
|
from buildbot.schedulers.basic import SingleBranchScheduler
|
|
from buildbot.schedulers.filter import ChangeFilter
|
|
from buildbot.changes.gitpoller import GitPoller
|
|
from buildbot.changes.svnpoller import SVNPoller
|
|
from buildbot.worker import Worker
|
|
|
|
from datetime import timedelta
|
|
|
|
# NOTE: We load the workers and their passwords from a separator file, so we can
|
|
# have # this one in Git.
|
|
import master_private
|
|
|
|
# Dictionary that the buildmaster pays attention to.
|
|
c = BuildmasterConfig = {}
|
|
|
|
# Project identity.
|
|
c['projectName'] = 'Blender'
|
|
c['projectURL'] = 'https://www.blender.org/'
|
|
c['title'] = 'Blender'
|
|
c['titleURL'] = 'https://builder.blender.org/'
|
|
|
|
# Buildbot information.
|
|
c['buildbotURL'] = 'https://builder.blender.org/admin/'
|
|
c['buildbotNetUsageData'] = 'basic'
|
|
|
|
# Various.
|
|
c['db_url'] = 'sqlite:///state.sqlite'
|
|
|
|
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
|
|
c['buildbotNetUsageData'] = None
|
|
|
|
################################################################################
|
|
# BUILD WORKERS
|
|
|
|
c['workers'] = []
|
|
for worker in master_private.workers:
|
|
c['workers'].append(
|
|
Worker(worker['name'], worker['password'], max_builds=1))
|
|
|
|
# TCP port through which workers connect
|
|
c['protocols'] = {
|
|
'pb': {
|
|
'port': 'tcp:{}'.format(9989)
|
|
}
|
|
}
|
|
|
|
################################################################################
|
|
# CODEBASES
|
|
#
|
|
# Allow to control separately things like branches for each repo and submodules.
|
|
|
|
all_repositories = {
|
|
r'git://git.blender.org/blender.git': 'blender',
|
|
r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
|
|
}
|
|
|
|
|
|
def codebaseGenerator(chdict):
|
|
return all_repositories[chdict['repository']]
|
|
|
|
|
|
c['codebaseGenerator'] = codebaseGenerator
|
|
|
|
|
|
################################################################################
|
|
# SCHEDULERS
|
|
#
|
|
# Decide how to react to incoming changes.
|
|
|
|
|
|
c['schedulers'] = []
|
|
|
|
|
|
def schedule_force_build(name, branch):
|
|
"""
|
|
Makes it possible to have "Force Build" for the given builder.
|
|
Makes sure only reasonable subset of properties are exposed.
|
|
"""
|
|
if branch != '':
|
|
branch_parameter = forcesched.FixedParameter(
|
|
name='branch',
|
|
default=branch,
|
|
hide=True)
|
|
else:
|
|
branch_parameter = forcesched.StringParameter(
|
|
name='branch',
|
|
label='Branch:',
|
|
default='custom-branch-name-here',
|
|
regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$')
|
|
|
|
c['schedulers'].append(forcesched.ForceScheduler(
|
|
name='force_' + name,
|
|
buttonName='Force Build',
|
|
builderNames=[name],
|
|
codebases=[forcesched.CodebaseParameter(
|
|
codebase='blender',
|
|
branch=branch_parameter,
|
|
# Hide revision. We don't want to allow anyone to overwrite the
|
|
# master build with an older version. Could be added back once we
|
|
# have authentication.
|
|
revision=forcesched.FixedParameter(
|
|
name='revision',
|
|
default='',
|
|
hide=True),
|
|
repository=forcesched.FixedParameter(
|
|
name='repository',
|
|
default='',
|
|
hide=True),
|
|
project=forcesched.FixedParameter(
|
|
name='project',
|
|
default='',
|
|
hide=True)),
|
|
]))
|
|
|
|
|
|
def schedule_nightly_build(name, branch, hour, minute=0):
|
|
"""
|
|
Creates scheduler for nightly builds for a given builder.
|
|
"""
|
|
scheduler_name = f'nightly_{name} {branch}'
|
|
c['schedulers'].append(timed.Nightly(
|
|
name=scheduler_name,
|
|
codebases={
|
|
'blender': {'repository': '',
|
|
'branch': branch}},
|
|
branch=branch,
|
|
builderNames=[name],
|
|
hour=hour,
|
|
minute=minute))
|
|
|
|
|
|
def schedule_change_build(name, branch):
|
|
"""
|
|
Creates scheduler for building on changes.
|
|
This will not package and upload the build.
|
|
"""
|
|
scheduler_name = f'change_{name} {branch}'
|
|
c['schedulers'].append(SingleBranchScheduler(
|
|
name=scheduler_name,
|
|
codebases={
|
|
'blender': {'repository': '',
|
|
'branch': branch}},
|
|
builderNames=[name],
|
|
treeStableTimer=120,
|
|
change_filter=ChangeFilter(project=['blender'], branch=branch),
|
|
properties={'skip_upload': True, 'skip_codesign': True}))
|
|
|
|
|
|
################################################################################
|
|
# BUILDERS
|
|
#
|
|
# The 'builders' list defines the Builders, which tell Buildbot how to
|
|
# perform a build: what steps, and which workers can execute them.
|
|
# Note that any particular build will only take place on one worker.
|
|
|
|
c['builders'] = []
|
|
builders_all_branches = set()
|
|
|
|
|
|
# Add builder utility.
|
|
|
|
def add_builder(c, name, platforms, factory, branch='', hour=3, minute=0, use_nightly_builder=True):
|
|
if branch != '':
|
|
builders_all_branches.add(branch)
|
|
|
|
for platform in platforms:
|
|
workernames = []
|
|
builder_name = f'{name}_{platform}'
|
|
|
|
for worker in master_private.workers:
|
|
if platform == worker['platform']:
|
|
workernames.append(worker['name'])
|
|
builder_name = f"{worker['platform_short']}_{name}"
|
|
|
|
if workernames:
|
|
f = factory(builder_name, branch)
|
|
c['builders'].append(BuilderConfig(name=builder_name,
|
|
workernames=workernames,
|
|
factory=f,
|
|
tags=['blender']))
|
|
|
|
if use_nightly_builder:
|
|
schedule_nightly_build(builder_name, branch, hour, minute)
|
|
schedule_change_build(builder_name, branch)
|
|
schedule_force_build(builder_name, branch)
|
|
|
|
|
|
# Common steps.
|
|
|
|
def git_step(branch=''):
|
|
if branch:
|
|
return steps.Git(name='blender.git',
|
|
repourl='git://git.blender.org/blender.git',
|
|
mode='incremental',
|
|
branch=branch,
|
|
codebase='blender',
|
|
workdir='blender.git',
|
|
submodules=True)
|
|
else:
|
|
return steps.Git(name='blender.git',
|
|
repourl='git://git.blender.org/blender.git',
|
|
mode='incremental',
|
|
codebase='blender',
|
|
workdir='blender.git',
|
|
submodules=True)
|
|
|
|
|
|
# Generic builder.
|
|
|
|
def do_upload(step):
|
|
return not step.hasProperty('skip_upload')
|
|
|
|
@util.renderer
|
|
def script_command(props, script, id, branch):
|
|
# NOTE: On Windows never includes major version in the executable name,
|
|
# so Python 3 will have be 'python.exe'.
|
|
if id.startswith('win'):
|
|
python_command = 'python'
|
|
else:
|
|
python_command = 'python3'
|
|
|
|
git_branch = branch or Interpolate('%(src:blender:branch)s')
|
|
args = [python_command, script, id, git_branch]
|
|
|
|
if not props.hasProperty('skip_codesign'):
|
|
args += ['--codesign']
|
|
|
|
return args
|
|
|
|
def generic_builder(id, branch=''):
|
|
# TODO(sergey): Consider using pathlib.
|
|
filename = f'uploaded/buildbot_upload_{id}.zip'
|
|
update_script = '../blender.git/build_files/buildbot/worker_update.py'
|
|
compile_script = '../blender.git/build_files/buildbot/worker_compile.py'
|
|
test_script = '../blender.git/build_files/buildbot/worker_test.py'
|
|
pack_script = '../blender.git/build_files/buildbot/worker_pack.py'
|
|
unpack_script = 'master_unpack.py'
|
|
|
|
f = BuildFactory()
|
|
f.addStep(git_step(branch))
|
|
|
|
f.addStep(ShellCommand(
|
|
name='submodules and libraries update',
|
|
command=script_command.withArgs(update_script, id, branch),
|
|
description='updating',
|
|
descriptionDone='updated',
|
|
haltOnFailure=True))
|
|
|
|
f.addStep(Compile(
|
|
command=script_command.withArgs(compile_script, id, branch),
|
|
timeout=3600))
|
|
f.addStep(Test(
|
|
command=script_command.withArgs(test_script, id, branch)))
|
|
f.addStep(ShellCommand(
|
|
name='package',
|
|
command=script_command.withArgs(pack_script, id, branch),
|
|
description='packaging',
|
|
descriptionDone='packaged',
|
|
haltOnFailure=True,
|
|
doStepIf=do_upload))
|
|
|
|
f.addStep(FileUpload(name='upload',
|
|
workersrc='buildbot_upload.zip',
|
|
masterdest=filename,
|
|
maxsize=500 * 1024 * 1024,
|
|
workdir='install',
|
|
doStepIf=do_upload))
|
|
f.addStep(MasterShellCommand(name='unpack',
|
|
command=['python3.7',
|
|
unpack_script,
|
|
filename],
|
|
description='unpacking',
|
|
descriptionDone='unpacked',
|
|
doStepIf=do_upload))
|
|
return f
|
|
|
|
|
|
################################################################################
|
|
# Builders
|
|
|
|
add_builder(c,
|
|
'master',
|
|
['windows', 'macOS_10_15', 'linux_centos7'],
|
|
generic_builder,
|
|
branch='master',
|
|
hour=1,
|
|
use_nightly_builder=True)
|
|
add_builder(c,
|
|
'292',
|
|
['windows', 'macOS_10_15', 'linux_centos7'],
|
|
generic_builder,
|
|
branch='blender-v2.92-release',
|
|
hour=1,
|
|
use_nightly_builder=False)
|
|
add_builder(c,
|
|
'lts_283',
|
|
['windows', 'macOS_10_13', 'linux_centos7'],
|
|
generic_builder,
|
|
branch='blender-v2.83-release',
|
|
hour=1,
|
|
use_nightly_builder=False)
|
|
add_builder(c,
|
|
'custom_branch',
|
|
['windows', 'macOS_10_15', 'linux_centos7'],
|
|
generic_builder,
|
|
branch='',
|
|
hour=1,
|
|
use_nightly_builder=False)
|
|
|
|
################################################################################
|
|
# CHANGE SOURCES
|
|
|
|
c['change_source'] = GitPoller(repourl='git://git.blender.org/blender.git',
|
|
pollinterval=120,
|
|
project='blender',
|
|
branches=list(builders_all_branches))
|
|
|
|
################################################################################
|
|
# HORIZONS
|
|
|
|
c['changeHorizon'] = 300
|
|
|
|
# Configure a janitor which will delete all logs older than one month,
|
|
# and will run on sundays at noon.
|
|
c['configurators'] = [util.JanitorConfigurator(
|
|
logHorizon=timedelta(weeks=4),
|
|
hour=12,
|
|
dayOfWeek=6)]
|
|
|
|
|
|
################################################################################
|
|
# WWW
|
|
|
|
c['www'] = dict(port=8010,
|
|
plugins={'console_view': {},
|
|
'grid_view': {},
|
|
'waterfall_view': {}})
|
|
|
|
################################################################################
|
|
# Access
|
|
|
|
|
|
class StrangerRoles(RolesFromBase):
|
|
def getRolesFromUser(self, userDetails):
|
|
return ['stranger']
|
|
|
|
|
|
authz = util.Authz(
|
|
# Simple matcher with '*' glob character.
|
|
# Could be util.reStrMatcher if regular expressions are preferred.
|
|
stringsMatcher=util.fnmatchStrMatcher,
|
|
|
|
allowRules=[
|
|
# Admins can do anything,
|
|
#
|
|
# defaultDeny=False: if user does not have the admin role, we continue
|
|
# parsing rules
|
|
util.AnyEndpointMatcher(role='admins', defaultDeny=False),
|
|
|
|
# Defaulting old config, everyone can stop build.
|
|
util.StopBuildEndpointMatcher(role='stranger'),
|
|
util.ForceBuildEndpointMatcher(role='stranger'),
|
|
util.RebuildBuildEndpointMatcher(role='stranger'),
|
|
util.EnableSchedulerEndpointMatcher(role='admins'),
|
|
|
|
# If future Buildbot implement new control, we are safe with this last
|
|
# rule.
|
|
#
|
|
# NOTE: This prevents us from cancelling queue, so disabled for now.
|
|
# util.AnyControlEndpointMatcher(role='admins')
|
|
],
|
|
roleMatchers=[
|
|
RolesFromOwner(role='owner'),
|
|
StrangerRoles(),
|
|
])
|
|
c['www']['authz'] = authz
|