419 lines
15 KiB
Python
419 lines
15 KiB
Python
# -*- python -*-
|
|
# ex: set syntax=python:
|
|
|
|
# <pep8 compliant>
|
|
|
|
# List of the branches being built automatically overnight
|
|
NIGHT_SCHEDULE_BRANCHES = [None, "blender2.7"]
|
|
|
|
# List of the branches available for force build
|
|
FORCE_SCHEDULE_BRANCHES = ["master", "blender2.7", "experimental-build"]
|
|
|
|
# List of branches availble for addons branch selection.
|
|
ADDONS_BRANCHES = ["master", "blender2.7"]
|
|
|
|
# List of branches availble for libraries branch selection.
|
|
LIB_BRANCHES = ["trunk"]
|
|
|
|
# Dictionary that the buildmaster pays attention to.
|
|
c = BuildmasterConfig = {}
|
|
|
|
# BUILD WORKERS
|
|
#
|
|
# We load the slaves and their passwords from a separator file, so we can have
|
|
# this one in SVN.
|
|
|
|
from buildbot.worker import Worker
|
|
import master_private
|
|
|
|
c['workers'] = []
|
|
for slave in master_private.slaves:
|
|
c['workers'].append(Worker(slave['name'], slave['password']))
|
|
|
|
# TCP port through which slaves connect
|
|
c['protocols'] = {
|
|
"pb": {
|
|
"port": "tcp:{}".format(9989)
|
|
}
|
|
}
|
|
# CHANGE SOURCES
|
|
|
|
from buildbot.changes.svnpoller import SVNPoller
|
|
from buildbot.changes.gitpoller import GitPoller
|
|
|
|
c['change_source'] = GitPoller('git://git.blender.org/blender.git',
|
|
pollinterval=1200)
|
|
|
|
# CODEBASES
|
|
#
|
|
# Allow to control separately things like branches for each repo and submodules.
|
|
|
|
all_repositories = {
|
|
r'git://git.blender.org/blender.git': 'blender',
|
|
r'git://git.blender.org/blender-translations.git': 'blender-translations',
|
|
r'git://git.blender.org/blender-addons.git': 'blender-addons',
|
|
r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
|
|
r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
|
|
r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
|
|
}
|
|
|
|
|
|
def codebaseGenerator(chdict):
|
|
return all_repositories[chdict['repository']]
|
|
|
|
|
|
c['codebaseGenerator'] = codebaseGenerator
|
|
|
|
|
|
# SCHEDULERS
|
|
#
|
|
# Decide how to react to incoming changes.
|
|
|
|
from buildbot.schedulers import timed, forcesched
|
|
|
|
c['schedulers'] = []
|
|
|
|
|
|
def schedule_force_build(name):
|
|
"""
|
|
Makes it possible to have "Force Build" for the given builder.
|
|
Makes sure only reasonabel subset of properties are exposed.
|
|
"""
|
|
c['schedulers'].append(forcesched.ForceScheduler(
|
|
name='force_' + name,
|
|
buttonName="Force Build",
|
|
builderNames=[name],
|
|
codebases=[forcesched.CodebaseParameter(
|
|
codebase="blender",
|
|
branch=forcesched.ChoiceStringParameter(
|
|
name="branch",
|
|
choices=FORCE_SCHEDULE_BRANCHES,
|
|
default="master"),
|
|
# Do not hide revision, can be handy!
|
|
repository=forcesched.FixedParameter(
|
|
name="repository",
|
|
default="",
|
|
hide=True),
|
|
project=forcesched.FixedParameter(
|
|
name="project",
|
|
default="",
|
|
hide=True)),
|
|
# For now, hide other codebases.
|
|
forcesched.CodebaseParameter(
|
|
hide=True,
|
|
codebase="blender-translations"),
|
|
forcesched.CodebaseParameter(
|
|
codebase="blender-addons",
|
|
branch=forcesched.ChoiceStringParameter(
|
|
name="branch",
|
|
choices=ADDONS_BRANCHES,
|
|
default="master"),
|
|
repository=forcesched.FixedParameter(name="repository",
|
|
default="",
|
|
hide=True),
|
|
project=forcesched.FixedParameter(name="project",
|
|
default="",
|
|
hide=True),
|
|
revision=forcesched.FixedParameter(name="revision",
|
|
default="",
|
|
hide=True),
|
|
),
|
|
forcesched.CodebaseParameter(
|
|
codebase="blender-addons-contrib",
|
|
branch=forcesched.ChoiceStringParameter(
|
|
name="branch",
|
|
choices=ADDONS_BRANCHES,
|
|
default="master"),
|
|
repository=forcesched.FixedParameter(name="repository",
|
|
default="",
|
|
hide=True),
|
|
project=forcesched.FixedParameter(name="project",
|
|
default="",
|
|
hide=True),
|
|
revision=forcesched.FixedParameter(name="revision",
|
|
default="",
|
|
hide=True),
|
|
),
|
|
forcesched.CodebaseParameter(
|
|
hide=True,
|
|
codebase="blender-dev-tools"),
|
|
forcesched.CodebaseParameter(
|
|
codebase="lib svn",
|
|
branch=forcesched.ChoiceStringParameter(
|
|
name="branch",
|
|
choices=LIB_BRANCHES,
|
|
default="trunk"),
|
|
repository=forcesched.FixedParameter(name="repository",
|
|
default="",
|
|
hide=True),
|
|
project=forcesched.FixedParameter(name="project",
|
|
default="",
|
|
hide=True),
|
|
revision=forcesched.FixedParameter(name="revision",
|
|
default="",
|
|
hide=True),
|
|
),
|
|
],
|
|
properties=[]))
|
|
|
|
|
|
def schedule_nightly_build(name, hour, minute=0):
|
|
"""
|
|
Creates scheduler for nightly builds for a given builder.
|
|
"""
|
|
for current_branch in NIGHT_SCHEDULE_BRANCHES:
|
|
scheduler_name = "nightly_" + name
|
|
if current_branch:
|
|
scheduler_name += ' ' + current_branch
|
|
# Use special addons submodule branch when building blender2.7 branch.
|
|
addons_branch = "master"
|
|
if current_branch == "blender2.7":
|
|
addons_branch = "blender2.7"
|
|
c['schedulers'].append(timed.Nightly(
|
|
name=scheduler_name,
|
|
codebases={
|
|
"blender": {"repository": "",
|
|
"branch": current_branch},
|
|
"blender-translations": {"repository": "",
|
|
"branch": "master"},
|
|
"blender-addons": {"repository": "",
|
|
"branch": addons_branch},
|
|
"blender-addons-contrib": {"repository": "",
|
|
"branch": addons_branch},
|
|
"blender-dev-tools": {"repository": "",
|
|
"branch": "master"},
|
|
"lib svn": {"repository": "",
|
|
"branch": "trunk"}},
|
|
branch=current_branch,
|
|
builderNames=[name],
|
|
hour=hour,
|
|
minute=minute))
|
|
|
|
|
|
# BUILDERS
|
|
#
|
|
# The 'builders' list defines the Builders, which tell Buildbot how to
|
|
# perform a build: what steps, and which slaves can execute them.
|
|
# Note that any particular build will only take place on one slave.
|
|
|
|
from buildbot.config import BuilderConfig
|
|
from buildbot.plugins import steps, util
|
|
from buildbot.process.factory import BuildFactory
|
|
from buildbot.process.properties import Interpolate
|
|
from buildbot.steps.shell import ShellCommand
|
|
from buildbot.steps.shell import Compile
|
|
from buildbot.steps.shell import Test
|
|
from buildbot.steps.transfer import FileUpload
|
|
from buildbot.steps.master import MasterShellCommand
|
|
|
|
# add builder utility
|
|
|
|
c['builders'] = []
|
|
buildernames = []
|
|
|
|
|
|
def add_builder(c, name, libdir, factory, branch='',
|
|
rsync=False, hour=3, minute=0):
|
|
workernames = []
|
|
|
|
for slave in master_private.slaves:
|
|
if name in slave['builders']:
|
|
workernames.append(slave['name'])
|
|
|
|
if workernames:
|
|
f = factory(name, libdir, branch, rsync)
|
|
c['builders'].append(BuilderConfig(name=name,
|
|
workernames=workernames,
|
|
factory=f,
|
|
tags=['blender']))
|
|
buildernames.append(name)
|
|
|
|
schedule_nightly_build(name, hour, minute)
|
|
schedule_force_build(name)
|
|
|
|
|
|
# common steps
|
|
|
|
def git_submodule_step(submodule):
|
|
return steps.Git(name=submodule + '.git',
|
|
repourl='git://git.blender.org/' + submodule + '.git',
|
|
mode='incremental',
|
|
codebase=submodule,
|
|
workdir=submodule + '.git')
|
|
|
|
|
|
def git_step(branch=''):
|
|
if branch:
|
|
return steps.Git(name='blender.git',
|
|
repourl='git://git.blender.org/blender.git',
|
|
mode='incremental',
|
|
branch=branch,
|
|
codebase='blender',
|
|
workdir='blender.git',
|
|
submodules=True)
|
|
else:
|
|
return steps.Git(name='blender.git',
|
|
repourl='git://git.blender.org/blender.git',
|
|
mode='incremental',
|
|
codebase='blender',
|
|
workdir='blender.git',
|
|
submodules=True)
|
|
|
|
|
|
def git_submodules_update():
|
|
command = ['git', 'submodule', 'update', '--remote']
|
|
return ShellCommand(name='Submodules Update',
|
|
command=command,
|
|
description='updating',
|
|
descriptionDone='up to date',
|
|
workdir='blender.git')
|
|
|
|
|
|
def lib_svn_step(dir):
|
|
# TODO(sergey): For some reason interpolation is always giving empty branch.
|
|
# lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
|
|
lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
|
|
return steps.SVN(name='lib svn',
|
|
repourl=util.Interpolate(lib_repo + dir),
|
|
codebase='lib svn',
|
|
mode='incremental',
|
|
workdir='lib/' + dir)
|
|
|
|
|
|
def rsync_step(id, branch, rsync_script):
|
|
return ShellCommand(name='rsync',
|
|
command=['python', rsync_script, id, branch],
|
|
description='uploading',
|
|
descriptionDone='uploaded',
|
|
workdir='install')
|
|
|
|
|
|
# generic builder
|
|
|
|
def generic_builder(id, libdir='', branch='', rsync=False):
|
|
filename = 'uploaded/buildbot_upload_' + id + '.zip'
|
|
compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
|
|
test_script = '../blender.git/build_files/buildbot/slave_test.py'
|
|
pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
|
|
rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
|
|
unpack_script = 'master_unpack.py'
|
|
|
|
f = BuildFactory()
|
|
if libdir != '':
|
|
f.addStep(lib_svn_step(libdir))
|
|
|
|
for submodule in ('blender-translations',
|
|
'blender-addons',
|
|
'blender-addons-contrib',
|
|
'blender-dev-tools'):
|
|
f.addStep(git_submodule_step(submodule))
|
|
|
|
f.addStep(git_step(branch))
|
|
f.addStep(git_submodules_update())
|
|
|
|
f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
|
|
f.addStep(Test(command=['python', test_script, id]))
|
|
f.addStep(ShellCommand(
|
|
name='package',
|
|
command=['python',
|
|
pack_script,
|
|
id,
|
|
branch or Interpolate('%(src:blender:branch)s')],
|
|
description='packaging',
|
|
descriptionDone='packaged'))
|
|
if rsync:
|
|
f.addStep(rsync_step(id, branch, rsync_script))
|
|
else:
|
|
f.addStep(FileUpload(name='upload',
|
|
workersrc='buildbot_upload.zip',
|
|
masterdest=filename,
|
|
maxsize=180 * 1024 * 1024,
|
|
workdir='install'))
|
|
f.addStep(MasterShellCommand(name='unpack',
|
|
command=['python2.7',
|
|
unpack_script,
|
|
filename],
|
|
description='unpacking',
|
|
descriptionDone='unpacked'))
|
|
return f
|
|
|
|
|
|
# Builders
|
|
|
|
add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
|
|
add_builder(c, 'linux_glibc224_i686_cmake', '', generic_builder, hour=2)
|
|
add_builder(c, 'linux_glibc224_x86_64_cmake', '', generic_builder, hour=1)
|
|
# NOTE: Visual Studio 2017 (vc15) is using libraries folder from
|
|
# Visual Studio 2015 (vc14)
|
|
add_builder(c, 'win32_cmake_vs2017', 'windows_vc14', generic_builder, hour=1)
|
|
add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=2)
|
|
|
|
# HORIZONS
|
|
from datetime import timedelta
|
|
|
|
c['changeHorizon'] = 300
|
|
|
|
# Configure a janitor which will delete all logs older than one month,
|
|
# and will run on sundays at noon.
|
|
c['configurators'] = [util.JanitorConfigurator(
|
|
logHorizon=timedelta(weeks=4),
|
|
hour=12,
|
|
dayOfWeek=6)]
|
|
|
|
|
|
# WWW
|
|
c['www'] = dict(port=8010,
|
|
plugins={'console_view': {},
|
|
'grid_view': {},
|
|
'waterfall_view': {}})
|
|
|
|
# Access
|
|
from buildbot.www.authz.roles import RolesFromOwner
|
|
from buildbot.www.authz.roles import RolesFromBase
|
|
|
|
class StrangerRoles(RolesFromBase):
|
|
def getRolesFromUser(self, userDetails):
|
|
return ['stranger']
|
|
|
|
|
|
authz = util.Authz(
|
|
stringsMatcher=util.fnmatchStrMatcher, # simple matcher with '*' glob character
|
|
# stringsMatcher = util.reStrMatcher, # if you prefer regular expressions
|
|
allowRules=[
|
|
# admins can do anything,
|
|
# defaultDeny=False: if user does not have the admin role, we continue parsing rules
|
|
util.AnyEndpointMatcher(role="admins", defaultDeny=False),
|
|
|
|
# Defaulting old config, everyone can stop build.
|
|
util.StopBuildEndpointMatcher(role="stranger"),
|
|
util.ForceBuildEndpointMatcher(role="stranger"),
|
|
util.RebuildBuildEndpointMatcher(role="stranger"),
|
|
util.EnableSchedulerEndpointMatcher(role="admins"),
|
|
|
|
# if future Buildbot implement new control, we are safe with this last rule
|
|
# NOTE: This prevents us from cancelling queue, so disabled for now.
|
|
# util.AnyControlEndpointMatcher(role="admins")
|
|
],
|
|
roleMatchers=[
|
|
RolesFromOwner(role="owner"),
|
|
StrangerRoles(),
|
|
])
|
|
c['www']['authz'] = authz
|
|
|
|
# PROJECT IDENTITY
|
|
c['projectName'] = "Blender"
|
|
c['projectURL'] = "https://www.blender.org"
|
|
|
|
# Buildbot information
|
|
c['buildbotURL'] = "https://builder.blender.org/admin/"
|
|
c['buildbotNetUsageData'] = 'basic'
|
|
|
|
# Various
|
|
c['db_url'] = "sqlite:///state.sqlite"
|
|
|
|
c['title'] = "Blender"
|
|
c['titleURL'] = "https://builder.blender.org/"
|
|
|
|
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
|
|
c['buildbotNetUsageData'] = None
|