archive/blender-buildbot
Archived
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Cleanup: Really make pep-8 complaint, fix terminology

Browse Source
This commit is contained in:
Sergey Sharybin
2020-06-16 10:02:30 +02:00
parent 5817ddd56a
commit ee68c15433
2 changed files with 136 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
master.cfg
View File

@@ -1,37 +1,90 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# -*- python -*- # -*- python -*-
# ex: set syntax=python: # ex: set syntax=python:
# <pep8 compliant> # <pep8 compliant>
from buildbot.www.authz.roles import RolesFromBase
from buildbot.www.authz.roles import RolesFromOwner
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.steps.shell import Test
from buildbot.steps.shell import Compile
from buildbot.steps.shell import ShellCommand
from buildbot.process.properties import Interpolate
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, util
from buildbot.config import BuilderConfig
from buildbot.schedulers import timed, forcesched
from buildbot.changes.gitpoller import GitPoller
from buildbot.changes.svnpoller import SVNPoller
from buildbot.worker import Worker
from datetime import timedelta
# NOTE: We load the workers and their passwords from a separator file, so we can
# have # this one in Git.
import master_private
# Dictionary that the buildmaster pays attention to. # Dictionary that the buildmaster pays attention to.
c = BuildmasterConfig = {} c = BuildmasterConfig = {}
# BUILD WORKERS # Project identity.
# c['projectName'] = 'Blender'
# We load the slaves and their passwords from a separator file, so we can have c['projectURL'] = 'https://www.blender.org/'
# this one in SVN. c['title'] = 'Blender'
c['titleURL'] = 'https://builder.blender.org/'
from buildbot.worker import Worker # Buildbot information.
import master_private c['buildbotURL'] = 'https://builder.blender.org/admin/'
c['buildbotNetUsageData'] = 'basic'
# Various.
c['db_url'] = 'sqlite:///state.sqlite'
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
c['buildbotNetUsageData'] = None
################################################################################
# BUILD WORKERS
c['workers'] = [] c['workers'] = []
for slave in master_private.slaves: for worker in master_private.workers:
c['workers'].append(Worker(slave['name'], slave['password'], max_builds=1)) c['workers'].append(
Worker(worker['name'], worker['password'], max_builds=1))
# TCP port through which slaves connect # TCP port through which workers connect
c['protocols'] = { c['protocols'] = {
"pb": { 'pb': {
"port": "tcp:{}".format(9989) 'port': 'tcp:{}'.format(9989)
} }
} }
# CHANGE SOURCES
from buildbot.changes.svnpoller import SVNPoller ################################################################################
from buildbot.changes.gitpoller import GitPoller # CHANGE SOURCES
c['change_source'] = GitPoller('git://git.blender.org/blender.git', c['change_source'] = GitPoller('git://git.blender.org/blender.git',
pollinterval=1200) pollinterval=1200)
################################################################################
# CODEBASES # CODEBASES
# #
# Allow to control separately things like branches for each repo and submodules. # Allow to control separately things like branches for each repo and submodules.
@@ -49,11 +102,11 @@ def codebaseGenerator(chdict):
c['codebaseGenerator'] = codebaseGenerator c['codebaseGenerator'] = codebaseGenerator
################################################################################
# SCHEDULERS # SCHEDULERS
# #
# Decide how to react to incoming changes. # Decide how to react to incoming changes.
from buildbot.schedulers import timed, forcesched
c['schedulers'] = [] c['schedulers'] = []
@@ -63,40 +116,40 @@ def schedule_force_build(name, branch):
Makes it possible to have "Force Build" for the given builder. Makes it possible to have "Force Build" for the given builder.
Makes sure only reasonable subset of properties are exposed. Makes sure only reasonable subset of properties are exposed.
""" """
if branch != "": if branch != '':
branch_parameter = forcesched.FixedParameter( branch_parameter = forcesched.FixedParameter(
name="branch", name='branch',
default=branch, default=branch,
hide=True) hide=True)
else: else:
branch_parameter = forcesched.StringParameter( branch_parameter = forcesched.StringParameter(
name="branch", name='branch',
label="Branch:", label='Branch:',
default="custom-branch-name-here", default='custom-branch-name-here',
regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$') regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$')
c['schedulers'].append(forcesched.ForceScheduler( c['schedulers'].append(forcesched.ForceScheduler(
name='force_' + name, name='force_' + name,
buttonName="Force Build", buttonName='Force Build',
builderNames=[name], builderNames=[name],
codebases=[forcesched.CodebaseParameter( codebases=[forcesched.CodebaseParameter(
codebase="blender", codebase='blender',
branch=branch_parameter, branch=branch_parameter,
# Hide revision. We don't want to allow anyone to overwrite the # Hide revision. We don't want to allow anyone to overwrite the
# master build with an older version. Could be added back once we # master build with an older version. Could be added back once we
# have authentication. # have authentication.
revision=forcesched.FixedParameter( revision=forcesched.FixedParameter(
name="revision", name='revision',
default="", default='',
hide=True), hide=True),
repository=forcesched.FixedParameter( repository=forcesched.FixedParameter(
name="repository", name='repository',
default="", default='',
hide=True), hide=True),
project=forcesched.FixedParameter( project=forcesched.FixedParameter(
name="project", name='project',
default="", default='',
hide=True)), hide=True)),
], ],
properties=[])) properties=[]))
@@ -105,49 +158,40 @@ def schedule_nightly_build(name, branch, hour, minute=0):
""" """
Creates scheduler for nightly builds for a given builder. Creates scheduler for nightly builds for a given builder.
""" """
scheduler_name = "nightly_" + name + ' ' + branch scheduler_name = f'nightly_{name} {branch}'
c['schedulers'].append(timed.Nightly( c['schedulers'].append(timed.Nightly(
name=scheduler_name, name=scheduler_name,
codebases={ codebases={
"blender": {"repository": "", 'blender': {'repository': '',
"branch": branch}}, 'branch': branch}},
branch=branch, branch=branch,
builderNames=[name], builderNames=[name],
hour=hour, hour=hour,
minute=minute)) minute=minute))
################################################################################
# BUILDERS # BUILDERS
# #
# The 'builders' list defines the Builders, which tell Buildbot how to # The 'builders' list defines the Builders, which tell Buildbot how to
# perform a build: what steps, and which slaves can execute them. # perform a build: what steps, and which workers can execute them.
# Note that any particular build will only take place on one slave. # Note that any particular build will only take place on one worker.
from buildbot.config import BuilderConfig
from buildbot.plugins import steps, util
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shell import Compile
from buildbot.steps.shell import Test
from buildbot.steps.transfer import FileUpload
from buildbot.steps.master import MasterShellCommand
# add builder utility
c['builders'] = [] c['builders'] = []
# Add builder utility.
def add_builder(c, name, platforms, factory, branch='', def add_builder(c, name, platforms, factory, branch='',
rsync=False, hour=3, minute=0): rsync=False, hour=3, minute=0):
for platform in platforms: for platform in platforms:
workernames = [] workernames = []
builder_name = name + '_' + platform builder_name = f'{name}_{platform}'
for slave in master_private.slaves: for worker in master_private.workers:
if platform == slave['platform']: if platform == worker['platform']:
workernames.append(slave['name']) workernames.append(worker['name'])
builder_name = slave['platform_short'] + '_' + name builder_name = f"{worker['platform_short']}_{name}"
if workernames: if workernames:
f = factory(builder_name, branch, rsync) f = factory(builder_name, branch, rsync)
@@ -161,7 +205,7 @@ def add_builder(c, name, platforms, factory, branch='',
schedule_force_build(builder_name, branch) schedule_force_build(builder_name, branch)
# common steps # Common steps.
def git_step(branch=''): def git_step(branch=''):
if branch: if branch:
@@ -190,10 +234,11 @@ def rsync_step(python_command, id, branch, rsync_script):
haltOnFailure=True) haltOnFailure=True)
# generic builder # Generic builder.
def generic_builder(id, branch='', rsync=False): def generic_builder(id, branch='', rsync=False):
filename = 'uploaded/buildbot_upload_' + id + '.zip' # TODO(sergey): Consider using pathlib.
filename = f'uploaded/buildbot_upload_{id}.zip'
update_script = '../blender.git/build_files/buildbot/slave_update.py' update_script = '../blender.git/build_files/buildbot/slave_update.py'
compile_script = '../blender.git/build_files/buildbot/slave_compile.py' compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
test_script = '../blender.git/build_files/buildbot/slave_test.py' test_script = '../blender.git/build_files/buildbot/slave_test.py'
@@ -201,12 +246,14 @@ def generic_builder(id, branch='', rsync=False):
rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py' rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
unpack_script = 'master_unpack.py' unpack_script = 'master_unpack.py'
f = BuildFactory() # NOTE: On Windows never includes major version in the executable name,
# so Python 3 will have be 'python.exe'.
if id.startswith('win'): if id.startswith('win'):
python_command = 'python' python_command = 'python'
else: else:
python_command = 'python3' python_command = 'python3'
f = BuildFactory()
f.addStep(git_step(branch)) f.addStep(git_step(branch))
git_branch = branch or Interpolate('%(src:blender:branch)s') git_branch = branch or Interpolate('%(src:blender:branch)s')
@@ -247,6 +294,7 @@ def generic_builder(id, branch='', rsync=False):
return f return f
################################################################################
# Builders # Builders
add_builder(c, add_builder(c,
@@ -268,8 +316,8 @@ add_builder(c,
branch='', branch='',
hour=1) hour=1)
################################################################################
# HORIZONS # HORIZONS
from datetime import timedelta
c['changeHorizon'] = 300 c['changeHorizon'] = 300
@@ -281,15 +329,17 @@ c['configurators'] = [util.JanitorConfigurator(
dayOfWeek=6)] dayOfWeek=6)]
################################################################################
# WWW # WWW
c['www'] = dict(port=8010, c['www'] = dict(port=8010,
plugins={'console_view': {}, plugins={'console_view': {},
'grid_view': {}, 'grid_view': {},
'waterfall_view': {}}) 'waterfall_view': {}})
################################################################################
# Access # Access
from buildbot.www.authz.roles import RolesFromOwner
from buildbot.www.authz.roles import RolesFromBase
class StrangerRoles(RolesFromBase): class StrangerRoles(RolesFromBase):
def getRolesFromUser(self, userDetails): def getRolesFromUser(self, userDetails):
@@ -297,42 +347,31 @@ class StrangerRoles(RolesFromBase):
authz = util.Authz( authz = util.Authz(
stringsMatcher=util.fnmatchStrMatcher, # simple matcher with '*' glob character # Simple matcher with '*' glob character.
# stringsMatcher = util.reStrMatcher, # if you prefer regular expressions # Could be util.reStrMatcher if regular expressions are preferred.
stringsMatcher=util.fnmatchStrMatcher,
allowRules=[ allowRules=[
# admins can do anything, # Admins can do anything,
# defaultDeny=False: if user does not have the admin role, we continue parsing rules #
util.AnyEndpointMatcher(role="admins", defaultDeny=False), # defaultDeny=False: if user does not have the admin role, we continue
# parsing rules
util.AnyEndpointMatcher(role='admins', defaultDeny=False),
# Defaulting old config, everyone can stop build. # Defaulting old config, everyone can stop build.
util.StopBuildEndpointMatcher(role="stranger"), util.StopBuildEndpointMatcher(role='stranger'),
util.ForceBuildEndpointMatcher(role="stranger"), util.ForceBuildEndpointMatcher(role='stranger'),
util.RebuildBuildEndpointMatcher(role="stranger"), util.RebuildBuildEndpointMatcher(role='stranger'),
util.EnableSchedulerEndpointMatcher(role="admins"), util.EnableSchedulerEndpointMatcher(role='admins'),
# if future Buildbot implement new control, we are safe with this last rule # If future Buildbot implement new control, we are safe with this last
# rule.
#
# NOTE: This prevents us from cancelling queue, so disabled for now. # NOTE: This prevents us from cancelling queue, so disabled for now.
# util.AnyControlEndpointMatcher(role="admins") # util.AnyControlEndpointMatcher(role='admins')
], ],
roleMatchers=[ roleMatchers=[
RolesFromOwner(role="owner"), RolesFromOwner(role='owner'),
StrangerRoles(), StrangerRoles(),
]) ])
c['www']['authz'] = authz c['www']['authz'] = authz
# PROJECT IDENTITY
c['projectName'] = "Blender"
c['projectURL'] = "https://www.blender.org"
# Buildbot information
c['buildbotURL'] = "https://builder.blender.org/admin/"
c['buildbotNetUsageData'] = 'basic'
# Various
c['db_url'] = "sqlite:///state.sqlite"
c['title'] = "Blender"
c['titleURL'] = "https://builder.blender.org/"
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
c['buildbotNetUsageData'] = None

2
master_private_template.py
View File

@@ -1,4 +1,4 @@
slaves = [ workers = [
{'name': 'linux_something_something', {'name': 'linux_something_something',
'password': 'something', 'password': 'something',
'platform': 'linux_centos7', 'platform': 'linux_centos7',