diff --git a/master.cfg b/master.cfg index cc49815..8883e37 100644 --- a/master.cfg +++ b/master.cfg @@ -1,37 +1,90 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + # -*- python -*- # ex: set syntax=python: # + +from buildbot.www.authz.roles import RolesFromBase +from buildbot.www.authz.roles import RolesFromOwner +from buildbot.steps.master import MasterShellCommand +from buildbot.steps.transfer import FileUpload +from buildbot.steps.shell import Test +from buildbot.steps.shell import Compile +from buildbot.steps.shell import ShellCommand +from buildbot.process.properties import Interpolate +from buildbot.process.factory import BuildFactory +from buildbot.plugins import steps, util +from buildbot.config import BuilderConfig +from buildbot.schedulers import timed, forcesched +from buildbot.changes.gitpoller import GitPoller +from buildbot.changes.svnpoller import SVNPoller +from buildbot.worker import Worker + +from datetime import timedelta + +# NOTE: We load the workers and their passwords from a separator file, so we can +# have # this one in Git. +import master_private + # Dictionary that the buildmaster pays attention to. c = BuildmasterConfig = {} -# BUILD WORKERS -# -# We load the slaves and their passwords from a separator file, so we can have -# this one in SVN. +# Project identity. +c['projectName'] = 'Blender' +c['projectURL'] = 'https://www.blender.org/' +c['title'] = 'Blender' +c['titleURL'] = 'https://builder.blender.org/' -from buildbot.worker import Worker -import master_private +# Buildbot information. +c['buildbotURL'] = 'https://builder.blender.org/admin/' +c['buildbotNetUsageData'] = 'basic' + +# Various. +c['db_url'] = 'sqlite:///state.sqlite' + +# Disable sending of 'buildbotNetUsageData' for now, to improve startup time. +c['buildbotNetUsageData'] = None + +################################################################################ +# BUILD WORKERS c['workers'] = [] -for slave in master_private.slaves: - c['workers'].append(Worker(slave['name'], slave['password'], max_builds=1)) +for worker in master_private.workers: + c['workers'].append( + Worker(worker['name'], worker['password'], max_builds=1)) -# TCP port through which slaves connect +# TCP port through which workers connect c['protocols'] = { - "pb": { - "port": "tcp:{}".format(9989) + 'pb': { + 'port': 'tcp:{}'.format(9989) } } -# CHANGE SOURCES -from buildbot.changes.svnpoller import SVNPoller -from buildbot.changes.gitpoller import GitPoller +################################################################################ +# CHANGE SOURCES c['change_source'] = GitPoller('git://git.blender.org/blender.git', pollinterval=1200) +################################################################################ # CODEBASES # # Allow to control separately things like branches for each repo and submodules. @@ -49,11 +102,11 @@ def codebaseGenerator(chdict): c['codebaseGenerator'] = codebaseGenerator +################################################################################ # SCHEDULERS # # Decide how to react to incoming changes. -from buildbot.schedulers import timed, forcesched c['schedulers'] = [] @@ -63,40 +116,40 @@ def schedule_force_build(name, branch): Makes it possible to have "Force Build" for the given builder. Makes sure only reasonable subset of properties are exposed. """ - if branch != "": + if branch != '': branch_parameter = forcesched.FixedParameter( - name="branch", + name='branch', default=branch, hide=True) else: branch_parameter = forcesched.StringParameter( - name="branch", - label="Branch:", - default="custom-branch-name-here", + name='branch', + label='Branch:', + default='custom-branch-name-here', regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$') c['schedulers'].append(forcesched.ForceScheduler( name='force_' + name, - buttonName="Force Build", + buttonName='Force Build', builderNames=[name], codebases=[forcesched.CodebaseParameter( - codebase="blender", - branch=branch_parameter, - # Hide revision. We don't want to allow anyone to overwrite the - # master build with an older version. Could be added back once we - # have authentication. - revision=forcesched.FixedParameter( - name="revision", - default="", - hide=True), - repository=forcesched.FixedParameter( - name="repository", - default="", - hide=True), - project=forcesched.FixedParameter( - name="project", - default="", - hide=True)), + codebase='blender', + branch=branch_parameter, + # Hide revision. We don't want to allow anyone to overwrite the + # master build with an older version. Could be added back once we + # have authentication. + revision=forcesched.FixedParameter( + name='revision', + default='', + hide=True), + repository=forcesched.FixedParameter( + name='repository', + default='', + hide=True), + project=forcesched.FixedParameter( + name='project', + default='', + hide=True)), ], properties=[])) @@ -105,49 +158,40 @@ def schedule_nightly_build(name, branch, hour, minute=0): """ Creates scheduler for nightly builds for a given builder. """ - scheduler_name = "nightly_" + name + ' ' + branch + scheduler_name = f'nightly_{name} {branch}' c['schedulers'].append(timed.Nightly( name=scheduler_name, codebases={ - "blender": {"repository": "", - "branch": branch}}, + 'blender': {'repository': '', + 'branch': branch}}, branch=branch, builderNames=[name], hour=hour, minute=minute)) +################################################################################ # BUILDERS # # The 'builders' list defines the Builders, which tell Buildbot how to -# perform a build: what steps, and which slaves can execute them. -# Note that any particular build will only take place on one slave. - -from buildbot.config import BuilderConfig -from buildbot.plugins import steps, util -from buildbot.process.factory import BuildFactory -from buildbot.process.properties import Interpolate -from buildbot.steps.shell import ShellCommand -from buildbot.steps.shell import Compile -from buildbot.steps.shell import Test -from buildbot.steps.transfer import FileUpload -from buildbot.steps.master import MasterShellCommand - -# add builder utility +# perform a build: what steps, and which workers can execute them. +# Note that any particular build will only take place on one worker. c['builders'] = [] +# Add builder utility. + def add_builder(c, name, platforms, factory, branch='', rsync=False, hour=3, minute=0): for platform in platforms: workernames = [] - builder_name = name + '_' + platform + builder_name = f'{name}_{platform}' - for slave in master_private.slaves: - if platform == slave['platform']: - workernames.append(slave['name']) - builder_name = slave['platform_short'] + '_' + name + for worker in master_private.workers: + if platform == worker['platform']: + workernames.append(worker['name']) + builder_name = f"{worker['platform_short']}_{name}" if workernames: f = factory(builder_name, branch, rsync) @@ -161,7 +205,7 @@ def add_builder(c, name, platforms, factory, branch='', schedule_force_build(builder_name, branch) -# common steps +# Common steps. def git_step(branch=''): if branch: @@ -190,10 +234,11 @@ def rsync_step(python_command, id, branch, rsync_script): haltOnFailure=True) -# generic builder +# Generic builder. def generic_builder(id, branch='', rsync=False): - filename = 'uploaded/buildbot_upload_' + id + '.zip' + # TODO(sergey): Consider using pathlib. + filename = f'uploaded/buildbot_upload_{id}.zip' update_script = '../blender.git/build_files/buildbot/slave_update.py' compile_script = '../blender.git/build_files/buildbot/slave_compile.py' test_script = '../blender.git/build_files/buildbot/slave_test.py' @@ -201,12 +246,14 @@ def generic_builder(id, branch='', rsync=False): rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py' unpack_script = 'master_unpack.py' - f = BuildFactory() + # NOTE: On Windows never includes major version in the executable name, + # so Python 3 will have be 'python.exe'. if id.startswith('win'): python_command = 'python' else: python_command = 'python3' + f = BuildFactory() f.addStep(git_step(branch)) git_branch = branch or Interpolate('%(src:blender:branch)s') @@ -247,6 +294,7 @@ def generic_builder(id, branch='', rsync=False): return f +################################################################################ # Builders add_builder(c, @@ -268,8 +316,8 @@ add_builder(c, branch='', hour=1) +################################################################################ # HORIZONS -from datetime import timedelta c['changeHorizon'] = 300 @@ -281,15 +329,17 @@ c['configurators'] = [util.JanitorConfigurator( dayOfWeek=6)] +################################################################################ # WWW + c['www'] = dict(port=8010, plugins={'console_view': {}, 'grid_view': {}, 'waterfall_view': {}}) +################################################################################ # Access -from buildbot.www.authz.roles import RolesFromOwner -from buildbot.www.authz.roles import RolesFromBase + class StrangerRoles(RolesFromBase): def getRolesFromUser(self, userDetails): @@ -297,42 +347,31 @@ class StrangerRoles(RolesFromBase): authz = util.Authz( - stringsMatcher=util.fnmatchStrMatcher, # simple matcher with '*' glob character - # stringsMatcher = util.reStrMatcher, # if you prefer regular expressions + # Simple matcher with '*' glob character. + # Could be util.reStrMatcher if regular expressions are preferred. + stringsMatcher=util.fnmatchStrMatcher, + allowRules=[ - # admins can do anything, - # defaultDeny=False: if user does not have the admin role, we continue parsing rules - util.AnyEndpointMatcher(role="admins", defaultDeny=False), + # Admins can do anything, + # + # defaultDeny=False: if user does not have the admin role, we continue + # parsing rules + util.AnyEndpointMatcher(role='admins', defaultDeny=False), # Defaulting old config, everyone can stop build. - util.StopBuildEndpointMatcher(role="stranger"), - util.ForceBuildEndpointMatcher(role="stranger"), - util.RebuildBuildEndpointMatcher(role="stranger"), - util.EnableSchedulerEndpointMatcher(role="admins"), + util.StopBuildEndpointMatcher(role='stranger'), + util.ForceBuildEndpointMatcher(role='stranger'), + util.RebuildBuildEndpointMatcher(role='stranger'), + util.EnableSchedulerEndpointMatcher(role='admins'), - # if future Buildbot implement new control, we are safe with this last rule + # If future Buildbot implement new control, we are safe with this last + # rule. + # # NOTE: This prevents us from cancelling queue, so disabled for now. - # util.AnyControlEndpointMatcher(role="admins") + # util.AnyControlEndpointMatcher(role='admins') ], roleMatchers=[ - RolesFromOwner(role="owner"), + RolesFromOwner(role='owner'), StrangerRoles(), ]) c['www']['authz'] = authz - -# PROJECT IDENTITY -c['projectName'] = "Blender" -c['projectURL'] = "https://www.blender.org" - -# Buildbot information -c['buildbotURL'] = "https://builder.blender.org/admin/" -c['buildbotNetUsageData'] = 'basic' - -# Various -c['db_url'] = "sqlite:///state.sqlite" - -c['title'] = "Blender" -c['titleURL'] = "https://builder.blender.org/" - -# Disable sending of 'buildbotNetUsageData' for now, to improve startup time. -c['buildbotNetUsageData'] = None diff --git a/master_private_template.py b/master_private_template.py index 1e67738..ba692f6 100644 --- a/master_private_template.py +++ b/master_private_template.py @@ -1,4 +1,4 @@ -slaves = [ +workers = [ {'name': 'linux_something_something', 'password': 'something', 'platform': 'linux_centos7',