Compare commits
5 Commits
master
...
tmp-releas
Author | SHA1 | Date | |
---|---|---|---|
03cf4c0896 | |||
50e508fd18 | |||
8349e52e4a | |||
ea17163d35 | |||
a3099f46a3 |
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"project_id" : "Infrastructure: Blender Buildbot",
|
|
||||||
"conduit_uri" : "https://developer.blender.org/",
|
|
||||||
"phabricator.uri" : "https://developer.blender.org/",
|
|
||||||
"git.default-relative-commit" : "origin/master",
|
|
||||||
"arc.land.update.default" : "rebase",
|
|
||||||
"arc.land.onto.default" : "master"
|
|
||||||
}
|
|
423
master.cfg
423
master.cfg
@@ -1,86 +1,40 @@
|
|||||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or
|
|
||||||
# modify it under the terms of the GNU General Public License
|
|
||||||
# as published by the Free Software Foundation; either version 2
|
|
||||||
# of the License, or (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program; if not, write to the Free Software Foundation,
|
|
||||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
||||||
#
|
|
||||||
# ##### END GPL LICENSE BLOCK #####
|
|
||||||
|
|
||||||
# -*- python -*-
|
# -*- python -*-
|
||||||
# ex: set syntax=python:
|
# ex: set syntax=python:
|
||||||
|
|
||||||
# <pep8 compliant>
|
# <pep8 compliant>
|
||||||
|
|
||||||
|
# List of the branches being built automatically overnight
|
||||||
from buildbot.www.authz.roles import RolesFromBase
|
NIGHT_SCHEDULE_BRANCHES = ["master"]
|
||||||
from buildbot.www.authz.roles import RolesFromOwner
|
|
||||||
from buildbot.steps.master import MasterShellCommand
|
|
||||||
from buildbot.steps.transfer import FileUpload
|
|
||||||
from buildbot.steps.shell import Test
|
|
||||||
from buildbot.steps.shell import Compile
|
|
||||||
from buildbot.steps.shell import ShellCommand
|
|
||||||
from buildbot.process.properties import Interpolate
|
|
||||||
from buildbot.process.factory import BuildFactory
|
|
||||||
from buildbot.plugins import steps, util
|
|
||||||
from buildbot.config import BuilderConfig
|
|
||||||
from buildbot.schedulers import timed, forcesched
|
|
||||||
from buildbot.schedulers.basic import SingleBranchScheduler
|
|
||||||
from buildbot.schedulers.filter import ChangeFilter
|
|
||||||
from buildbot.changes.gitpoller import GitPoller
|
|
||||||
from buildbot.changes.svnpoller import SVNPoller
|
|
||||||
from buildbot.worker import Worker
|
|
||||||
|
|
||||||
from datetime import timedelta
|
|
||||||
|
|
||||||
# NOTE: We load the workers and their passwords from a separator file, so we can
|
|
||||||
# have # this one in Git.
|
|
||||||
import master_private
|
|
||||||
|
|
||||||
# Dictionary that the buildmaster pays attention to.
|
# Dictionary that the buildmaster pays attention to.
|
||||||
c = BuildmasterConfig = {}
|
c = BuildmasterConfig = {}
|
||||||
|
|
||||||
# Project identity.
|
|
||||||
c['projectName'] = 'Blender'
|
|
||||||
c['projectURL'] = 'https://www.blender.org/'
|
|
||||||
c['title'] = 'Blender'
|
|
||||||
c['titleURL'] = 'https://builder.blender.org/'
|
|
||||||
|
|
||||||
# Buildbot information.
|
|
||||||
c['buildbotURL'] = 'https://builder.blender.org/admin/'
|
|
||||||
c['buildbotNetUsageData'] = 'basic'
|
|
||||||
|
|
||||||
# Various.
|
|
||||||
c['db_url'] = 'sqlite:///state.sqlite'
|
|
||||||
|
|
||||||
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
|
|
||||||
c['buildbotNetUsageData'] = None
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# BUILD WORKERS
|
# BUILD WORKERS
|
||||||
|
#
|
||||||
|
# We load the slaves and their passwords from a separator file, so we can have
|
||||||
|
# this one in SVN.
|
||||||
|
|
||||||
|
from buildbot.worker import Worker
|
||||||
|
import master_private
|
||||||
|
|
||||||
c['workers'] = []
|
c['workers'] = []
|
||||||
for worker in master_private.workers:
|
for slave in master_private.slaves:
|
||||||
c['workers'].append(
|
c['workers'].append(Worker(slave['name'], slave['password']))
|
||||||
Worker(worker['name'], worker['password'], max_builds=1))
|
|
||||||
|
|
||||||
# TCP port through which workers connect
|
# TCP port through which slaves connect
|
||||||
c['protocols'] = {
|
c['protocols'] = {
|
||||||
'pb': {
|
"pb": {
|
||||||
'port': 'tcp:{}'.format(9989)
|
"port": "tcp:{}".format(9989)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
# CHANGE SOURCES
|
||||||
|
|
||||||
|
from buildbot.changes.svnpoller import SVNPoller
|
||||||
|
from buildbot.changes.gitpoller import GitPoller
|
||||||
|
|
||||||
|
c['change_source'] = GitPoller('git://git.blender.org/blender.git',
|
||||||
|
pollinterval=1200)
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# CODEBASES
|
# CODEBASES
|
||||||
#
|
#
|
||||||
# Allow to control separately things like branches for each repo and submodules.
|
# Allow to control separately things like branches for each repo and submodules.
|
||||||
@@ -98,130 +52,111 @@ def codebaseGenerator(chdict):
|
|||||||
c['codebaseGenerator'] = codebaseGenerator
|
c['codebaseGenerator'] = codebaseGenerator
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# SCHEDULERS
|
# SCHEDULERS
|
||||||
#
|
#
|
||||||
# Decide how to react to incoming changes.
|
# Decide how to react to incoming changes.
|
||||||
|
|
||||||
|
from buildbot.schedulers import timed, forcesched
|
||||||
|
|
||||||
c['schedulers'] = []
|
c['schedulers'] = []
|
||||||
|
|
||||||
|
|
||||||
def schedule_force_build(name, branch):
|
def schedule_force_build(name):
|
||||||
"""
|
"""
|
||||||
Makes it possible to have "Force Build" for the given builder.
|
Makes it possible to have "Force Build" for the given builder.
|
||||||
Makes sure only reasonable subset of properties are exposed.
|
Makes sure only reasonabel subset of properties are exposed.
|
||||||
"""
|
"""
|
||||||
if branch != '':
|
|
||||||
branch_parameter = forcesched.FixedParameter(
|
|
||||||
name='branch',
|
|
||||||
default=branch,
|
|
||||||
hide=True)
|
|
||||||
else:
|
|
||||||
branch_parameter = forcesched.StringParameter(
|
|
||||||
name='branch',
|
|
||||||
label='Branch:',
|
|
||||||
default='custom-branch-name-here',
|
|
||||||
regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$')
|
|
||||||
|
|
||||||
c['schedulers'].append(forcesched.ForceScheduler(
|
c['schedulers'].append(forcesched.ForceScheduler(
|
||||||
name='force_' + name,
|
name='force_' + name,
|
||||||
buttonName='Force Build',
|
buttonName="Force Build",
|
||||||
builderNames=[name],
|
builderNames=[name],
|
||||||
codebases=[forcesched.CodebaseParameter(
|
codebases=[forcesched.CodebaseParameter(
|
||||||
codebase='blender',
|
codebase="blender",
|
||||||
branch=branch_parameter,
|
branch=forcesched.StringParameter(
|
||||||
# Hide revision. We don't want to allow anyone to overwrite the
|
name="branch",
|
||||||
# master build with an older version. Could be added back once we
|
label="Branch:",
|
||||||
# have authentication.
|
default="master"),
|
||||||
revision=forcesched.FixedParameter(
|
# Hide revision. We don't want to allow anyone to overwrite the
|
||||||
name='revision',
|
# master build with an older version. Could be added back once we
|
||||||
default='',
|
# have authentication.
|
||||||
hide=True),
|
revision=forcesched.FixedParameter(
|
||||||
repository=forcesched.FixedParameter(
|
name="revision",
|
||||||
name='repository',
|
default="",
|
||||||
default='',
|
hide=True),
|
||||||
hide=True),
|
repository=forcesched.FixedParameter(
|
||||||
project=forcesched.FixedParameter(
|
name="repository",
|
||||||
name='project',
|
default="",
|
||||||
default='',
|
hide=True),
|
||||||
hide=True)),
|
project=forcesched.FixedParameter(
|
||||||
]))
|
name="project",
|
||||||
|
default="",
|
||||||
|
hide=True)),
|
||||||
|
],
|
||||||
|
properties=[]))
|
||||||
|
|
||||||
|
|
||||||
def schedule_nightly_build(name, branch, hour, minute=0):
|
def schedule_nightly_build(name, hour, minute=0):
|
||||||
"""
|
"""
|
||||||
Creates scheduler for nightly builds for a given builder.
|
Creates scheduler for nightly builds for a given builder.
|
||||||
"""
|
"""
|
||||||
scheduler_name = f'nightly_{name} {branch}'
|
for current_branch in NIGHT_SCHEDULE_BRANCHES:
|
||||||
c['schedulers'].append(timed.Nightly(
|
scheduler_name = "nightly_" + name
|
||||||
name=scheduler_name,
|
if current_branch:
|
||||||
codebases={
|
scheduler_name += ' ' + current_branch
|
||||||
'blender': {'repository': '',
|
c['schedulers'].append(timed.Nightly(
|
||||||
'branch': branch}},
|
name=scheduler_name,
|
||||||
branch=branch,
|
codebases={
|
||||||
builderNames=[name],
|
"blender": {"repository": "",
|
||||||
hour=hour,
|
"branch": current_branch}},
|
||||||
minute=minute))
|
branch=current_branch,
|
||||||
|
builderNames=[name],
|
||||||
|
hour=hour,
|
||||||
|
minute=minute))
|
||||||
|
|
||||||
|
|
||||||
def schedule_change_build(name, branch):
|
|
||||||
"""
|
|
||||||
Creates scheduler for building on changes.
|
|
||||||
This will not package and upload the build.
|
|
||||||
"""
|
|
||||||
scheduler_name = f'change_{name} {branch}'
|
|
||||||
c['schedulers'].append(SingleBranchScheduler(
|
|
||||||
name=scheduler_name,
|
|
||||||
codebases={
|
|
||||||
'blender': {'repository': '',
|
|
||||||
'branch': branch}},
|
|
||||||
builderNames=[name],
|
|
||||||
treeStableTimer=120,
|
|
||||||
change_filter=ChangeFilter(project=['blender'], branch=branch),
|
|
||||||
properties={'skip_upload': True, 'skip_codesign': True}))
|
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# BUILDERS
|
# BUILDERS
|
||||||
#
|
#
|
||||||
# The 'builders' list defines the Builders, which tell Buildbot how to
|
# The 'builders' list defines the Builders, which tell Buildbot how to
|
||||||
# perform a build: what steps, and which workers can execute them.
|
# perform a build: what steps, and which slaves can execute them.
|
||||||
# Note that any particular build will only take place on one worker.
|
# Note that any particular build will only take place on one slave.
|
||||||
|
|
||||||
|
from buildbot.config import BuilderConfig
|
||||||
|
from buildbot.plugins import steps, util
|
||||||
|
from buildbot.process.factory import BuildFactory
|
||||||
|
from buildbot.process.properties import Interpolate
|
||||||
|
from buildbot.steps.shell import ShellCommand
|
||||||
|
from buildbot.steps.shell import Compile
|
||||||
|
from buildbot.steps.shell import Test
|
||||||
|
from buildbot.steps.transfer import FileUpload
|
||||||
|
from buildbot.steps.master import MasterShellCommand
|
||||||
|
|
||||||
|
# add builder utility
|
||||||
|
|
||||||
c['builders'] = []
|
c['builders'] = []
|
||||||
builders_all_branches = set()
|
buildernames = []
|
||||||
|
|
||||||
|
|
||||||
# Add builder utility.
|
def add_builder(c, name, factory, branch='',
|
||||||
|
rsync=False, hour=3, minute=0):
|
||||||
|
workernames = []
|
||||||
|
|
||||||
def add_builder(c, name, platforms, factory, branch='', hour=3, minute=0, use_nightly_builder=True):
|
for slave in master_private.slaves:
|
||||||
if branch != '':
|
if name in slave['builders']:
|
||||||
builders_all_branches.add(branch)
|
workernames.append(slave['name'])
|
||||||
|
|
||||||
for platform in platforms:
|
if workernames:
|
||||||
workernames = []
|
f = factory(name, branch, rsync)
|
||||||
builder_name = f'{name}_{platform}'
|
c['builders'].append(BuilderConfig(name=name,
|
||||||
|
workernames=workernames,
|
||||||
|
factory=f,
|
||||||
|
tags=['blender']))
|
||||||
|
buildernames.append(name)
|
||||||
|
|
||||||
for worker in master_private.workers:
|
schedule_nightly_build(name, hour, minute)
|
||||||
if platform == worker['platform']:
|
schedule_force_build(name)
|
||||||
workernames.append(worker['name'])
|
|
||||||
builder_name = f"{worker['platform_short']}_{name}"
|
|
||||||
|
|
||||||
if workernames:
|
|
||||||
f = factory(builder_name, branch)
|
|
||||||
c['builders'].append(BuilderConfig(name=builder_name,
|
|
||||||
workernames=workernames,
|
|
||||||
factory=f,
|
|
||||||
tags=['blender']))
|
|
||||||
|
|
||||||
if use_nightly_builder:
|
|
||||||
schedule_nightly_build(builder_name, branch, hour, minute)
|
|
||||||
schedule_change_build(builder_name, branch)
|
|
||||||
schedule_force_build(builder_name, branch)
|
|
||||||
|
|
||||||
|
|
||||||
# Common steps.
|
# common steps
|
||||||
|
|
||||||
def git_step(branch=''):
|
def git_step(branch=''):
|
||||||
if branch:
|
if branch:
|
||||||
@@ -241,118 +176,79 @@ def git_step(branch=''):
|
|||||||
submodules=True)
|
submodules=True)
|
||||||
|
|
||||||
|
|
||||||
# Generic builder.
|
def rsync_step(python_command, id, branch, rsync_script):
|
||||||
|
return ShellCommand(name='rsync',
|
||||||
|
command=[python_command, rsync_script, id, branch],
|
||||||
|
description='uploading',
|
||||||
|
descriptionDone='uploaded',
|
||||||
|
workdir='install')
|
||||||
|
|
||||||
def do_upload(step):
|
|
||||||
return not step.hasProperty('skip_upload')
|
|
||||||
|
|
||||||
@util.renderer
|
# generic builder
|
||||||
def script_command(props, script, id, branch):
|
|
||||||
# NOTE: On Windows never includes major version in the executable name,
|
def generic_builder(id, branch='', rsync=False):
|
||||||
# so Python 3 will have be 'python.exe'.
|
filename = 'uploaded/buildbot_upload_' + id + '.zip'
|
||||||
|
update_script = '../blender.git/build_files/buildbot/slave_update.py'
|
||||||
|
compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
|
||||||
|
test_script = '../blender.git/build_files/buildbot/slave_test.py'
|
||||||
|
pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
|
||||||
|
rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
|
||||||
|
unpack_script = 'master_unpack.py'
|
||||||
|
|
||||||
|
f = BuildFactory()
|
||||||
if id.startswith('win'):
|
if id.startswith('win'):
|
||||||
python_command = 'python'
|
python_command = 'python'
|
||||||
else:
|
else:
|
||||||
python_command = 'python3'
|
python_command = 'python3'
|
||||||
|
|
||||||
git_branch = branch or Interpolate('%(src:blender:branch)s')
|
|
||||||
args = [python_command, script, id, git_branch]
|
|
||||||
|
|
||||||
if not props.hasProperty('skip_codesign'):
|
|
||||||
args += ['--codesign']
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
||||||
def generic_builder(id, branch=''):
|
|
||||||
# TODO(sergey): Consider using pathlib.
|
|
||||||
filename = f'uploaded/buildbot_upload_{id}.zip'
|
|
||||||
update_script = '../blender.git/build_files/buildbot/worker_update.py'
|
|
||||||
compile_script = '../blender.git/build_files/buildbot/worker_compile.py'
|
|
||||||
test_script = '../blender.git/build_files/buildbot/worker_test.py'
|
|
||||||
pack_script = '../blender.git/build_files/buildbot/worker_pack.py'
|
|
||||||
unpack_script = 'master_unpack.py'
|
|
||||||
|
|
||||||
f = BuildFactory()
|
|
||||||
f.addStep(git_step(branch))
|
f.addStep(git_step(branch))
|
||||||
|
|
||||||
|
git_branch = branch or Interpolate('%(src:blender:branch)s')
|
||||||
|
|
||||||
f.addStep(ShellCommand(
|
f.addStep(ShellCommand(
|
||||||
name='submodules and libraries update',
|
name='submodules and libraries update',
|
||||||
command=script_command.withArgs(update_script, id, branch),
|
command=[python_command, update_script, id, git_branch],
|
||||||
description='updating',
|
description='updating',
|
||||||
descriptionDone='updated',
|
descriptionDone='updated'))
|
||||||
haltOnFailure=True))
|
|
||||||
|
|
||||||
f.addStep(Compile(
|
f.addStep(Compile(
|
||||||
command=script_command.withArgs(compile_script, id, branch),
|
command=[python_command, compile_script, id, git_branch],
|
||||||
timeout=3600))
|
timeout=3600))
|
||||||
f.addStep(Test(
|
f.addStep(Test(
|
||||||
command=script_command.withArgs(test_script, id, branch)))
|
command=[python_command, test_script, id, git_branch]))
|
||||||
f.addStep(ShellCommand(
|
f.addStep(ShellCommand(
|
||||||
name='package',
|
name='package',
|
||||||
command=script_command.withArgs(pack_script, id, branch),
|
command=[python_command, pack_script, id, git_branch],
|
||||||
description='packaging',
|
description='packaging',
|
||||||
descriptionDone='packaged',
|
descriptionDone='packaged'))
|
||||||
haltOnFailure=True,
|
|
||||||
doStepIf=do_upload))
|
|
||||||
|
|
||||||
f.addStep(FileUpload(name='upload',
|
if rsync:
|
||||||
workersrc='buildbot_upload.zip',
|
f.addStep(rsync_step(python_command, id, git_branch, rsync_script))
|
||||||
masterdest=filename,
|
else:
|
||||||
maxsize=500 * 1024 * 1024,
|
f.addStep(FileUpload(name='upload',
|
||||||
workdir='install',
|
workersrc='buildbot_upload.zip',
|
||||||
doStepIf=do_upload))
|
masterdest=filename,
|
||||||
|
maxsize=180 * 1024 * 1024,
|
||||||
|
workdir='install'))
|
||||||
f.addStep(MasterShellCommand(name='unpack',
|
f.addStep(MasterShellCommand(name='unpack',
|
||||||
command=['python3.7',
|
command=['python3.6',
|
||||||
unpack_script,
|
unpack_script,
|
||||||
filename],
|
filename],
|
||||||
description='unpacking',
|
description='unpacking',
|
||||||
descriptionDone='unpacked',
|
descriptionDone='unpacked'))
|
||||||
doStepIf=do_upload))
|
|
||||||
return f
|
return f
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# Builders
|
# Builders
|
||||||
|
|
||||||
add_builder(c,
|
add_builder(c, 'mac_x86_64_10_9_cmake', generic_builder, hour=1)
|
||||||
'master',
|
add_builder(c, 'linux_glibc217_x86_64_cmake', generic_builder, hour=1)
|
||||||
['windows', 'macOS_10_15', 'linux_centos7'],
|
# NOTE: Visual Studio 2017 (vc15) is using libraries folder from
|
||||||
generic_builder,
|
# Visual Studio 2015 (vc14)
|
||||||
branch='master',
|
add_builder(c, 'win64_cmake_vs2017', generic_builder, hour=1)
|
||||||
hour=1,
|
|
||||||
use_nightly_builder=True)
|
|
||||||
add_builder(c,
|
|
||||||
'292',
|
|
||||||
['windows', 'macOS_10_15', 'linux_centos7'],
|
|
||||||
generic_builder,
|
|
||||||
branch='blender-v2.92-release',
|
|
||||||
hour=1,
|
|
||||||
use_nightly_builder=False)
|
|
||||||
add_builder(c,
|
|
||||||
'lts_283',
|
|
||||||
['windows', 'macOS_10_13', 'linux_centos7'],
|
|
||||||
generic_builder,
|
|
||||||
branch='blender-v2.83-release',
|
|
||||||
hour=1,
|
|
||||||
use_nightly_builder=False)
|
|
||||||
add_builder(c,
|
|
||||||
'custom_branch',
|
|
||||||
['windows', 'macOS_10_15', 'linux_centos7'],
|
|
||||||
generic_builder,
|
|
||||||
branch='',
|
|
||||||
hour=1,
|
|
||||||
use_nightly_builder=False)
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# CHANGE SOURCES
|
|
||||||
|
|
||||||
c['change_source'] = GitPoller(repourl='git://git.blender.org/blender.git',
|
|
||||||
pollinterval=120,
|
|
||||||
project='blender',
|
|
||||||
branches=list(builders_all_branches))
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# HORIZONS
|
# HORIZONS
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
c['changeHorizon'] = 300
|
c['changeHorizon'] = 300
|
||||||
|
|
||||||
@@ -364,17 +260,15 @@ c['configurators'] = [util.JanitorConfigurator(
|
|||||||
dayOfWeek=6)]
|
dayOfWeek=6)]
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# WWW
|
# WWW
|
||||||
|
|
||||||
c['www'] = dict(port=8010,
|
c['www'] = dict(port=8010,
|
||||||
plugins={'console_view': {},
|
plugins={'console_view': {},
|
||||||
'grid_view': {},
|
'grid_view': {},
|
||||||
'waterfall_view': {}})
|
'waterfall_view': {}})
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# Access
|
# Access
|
||||||
|
from buildbot.www.authz.roles import RolesFromOwner
|
||||||
|
from buildbot.www.authz.roles import RolesFromBase
|
||||||
|
|
||||||
class StrangerRoles(RolesFromBase):
|
class StrangerRoles(RolesFromBase):
|
||||||
def getRolesFromUser(self, userDetails):
|
def getRolesFromUser(self, userDetails):
|
||||||
@@ -382,31 +276,42 @@ class StrangerRoles(RolesFromBase):
|
|||||||
|
|
||||||
|
|
||||||
authz = util.Authz(
|
authz = util.Authz(
|
||||||
# Simple matcher with '*' glob character.
|
stringsMatcher=util.fnmatchStrMatcher, # simple matcher with '*' glob character
|
||||||
# Could be util.reStrMatcher if regular expressions are preferred.
|
# stringsMatcher = util.reStrMatcher, # if you prefer regular expressions
|
||||||
stringsMatcher=util.fnmatchStrMatcher,
|
|
||||||
|
|
||||||
allowRules=[
|
allowRules=[
|
||||||
# Admins can do anything,
|
# admins can do anything,
|
||||||
#
|
# defaultDeny=False: if user does not have the admin role, we continue parsing rules
|
||||||
# defaultDeny=False: if user does not have the admin role, we continue
|
util.AnyEndpointMatcher(role="admins", defaultDeny=False),
|
||||||
# parsing rules
|
|
||||||
util.AnyEndpointMatcher(role='admins', defaultDeny=False),
|
|
||||||
|
|
||||||
# Defaulting old config, everyone can stop build.
|
# Defaulting old config, everyone can stop build.
|
||||||
util.StopBuildEndpointMatcher(role='stranger'),
|
util.StopBuildEndpointMatcher(role="stranger"),
|
||||||
util.ForceBuildEndpointMatcher(role='stranger'),
|
util.ForceBuildEndpointMatcher(role="stranger"),
|
||||||
util.RebuildBuildEndpointMatcher(role='stranger'),
|
util.RebuildBuildEndpointMatcher(role="stranger"),
|
||||||
util.EnableSchedulerEndpointMatcher(role='admins'),
|
util.EnableSchedulerEndpointMatcher(role="admins"),
|
||||||
|
|
||||||
# If future Buildbot implement new control, we are safe with this last
|
# if future Buildbot implement new control, we are safe with this last rule
|
||||||
# rule.
|
|
||||||
#
|
|
||||||
# NOTE: This prevents us from cancelling queue, so disabled for now.
|
# NOTE: This prevents us from cancelling queue, so disabled for now.
|
||||||
# util.AnyControlEndpointMatcher(role='admins')
|
# util.AnyControlEndpointMatcher(role="admins")
|
||||||
],
|
],
|
||||||
roleMatchers=[
|
roleMatchers=[
|
||||||
RolesFromOwner(role='owner'),
|
RolesFromOwner(role="owner"),
|
||||||
StrangerRoles(),
|
StrangerRoles(),
|
||||||
])
|
])
|
||||||
c['www']['authz'] = authz
|
c['www']['authz'] = authz
|
||||||
|
|
||||||
|
# PROJECT IDENTITY
|
||||||
|
c['projectName'] = "Blender"
|
||||||
|
c['projectURL'] = "https://www.blender.org"
|
||||||
|
|
||||||
|
# Buildbot information
|
||||||
|
c['buildbotURL'] = "https://builder.blender.org/admin/"
|
||||||
|
c['buildbotNetUsageData'] = 'basic'
|
||||||
|
|
||||||
|
# Various
|
||||||
|
c['db_url'] = "sqlite:///state.sqlite"
|
||||||
|
|
||||||
|
c['title'] = "Blender"
|
||||||
|
c['titleURL'] = "https://builder.blender.org/"
|
||||||
|
|
||||||
|
# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
|
||||||
|
c['buildbotNetUsageData'] = None
|
||||||
|
@@ -1,18 +1,5 @@
|
|||||||
workers = [
|
slaves = [
|
||||||
{'name': 'linux_something_something',
|
{'name': 'linux_glibc219_x86_64_chroot',
|
||||||
'password': 'something',
|
'password': 'barbarianna',
|
||||||
'platform': 'linux_centos7',
|
'builders': ['linux_glibc219_x86_64_cmake']}
|
||||||
'platform_short': 'linux'},
|
|
||||||
{'name': 'macOS_10_15_something_something',
|
|
||||||
'password': 'something',
|
|
||||||
'platform': 'macOS_10_15',
|
|
||||||
'platform_short': 'macos'},
|
|
||||||
{'name': 'macOS_10_13_something_something',
|
|
||||||
'password': 'something',
|
|
||||||
'platform': 'macOS_10_13',
|
|
||||||
'platform_short': 'macos'},
|
|
||||||
{'name': 'windows_something_something',
|
|
||||||
'password': 'something',
|
|
||||||
'platform': 'windows',
|
|
||||||
'platform_short': 'windows'}
|
|
||||||
]
|
]
|
||||||
|
147
master_unpack.py
147
master_unpack.py
@@ -25,83 +25,79 @@ import argparse
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
DOWNLOAD_DIR = "/data/www/vhosts/builder.blender.org/webroot/download/"
|
|
||||||
|
|
||||||
# Parse package filename to extract branch and platform
|
# Parse package filename to extract branch and platform
|
||||||
class Package:
|
class Package:
|
||||||
def __init__(self, zipname):
|
def __init__(self, zipname):
|
||||||
self.zipname = zipname
|
self.zipname = zipname
|
||||||
self.filename = os.path.basename(zipname)
|
self.filename = os.path.basename(zipname)
|
||||||
self.version = ""
|
self.platform = self._get_platform(self.filename)
|
||||||
self.branch = ""
|
self.branch = self._get_branch(self.filename)
|
||||||
self.platform = ""
|
|
||||||
self.parsed = False
|
|
||||||
self.parse_filename()
|
|
||||||
|
|
||||||
# extension stripping
|
# extension stripping
|
||||||
def _strip_extension(self, filename):
|
def _strip_extension(self, filename):
|
||||||
filename, _ = os.path.splitext(filename)
|
extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
|
||||||
if filename.endswith('.tar'):
|
|
||||||
filename, _ = os.path.splitext(filename)
|
for ext in extensions:
|
||||||
|
if filename.endswith(ext):
|
||||||
|
filename = filename[:-len(ext)]
|
||||||
|
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
# extract platform from package name
|
# extract platform from package name
|
||||||
def parse_filename(self):
|
def _get_platform(self, filename):
|
||||||
# Name is one one of:
|
# name is blender-version-platform.extension. we want to get the
|
||||||
# blender-version-platform: official release
|
# platform out, but there may be some variations, so we fiddle a
|
||||||
# blender-version-hash-platform: daily build
|
# bit to handle current and hopefully future names
|
||||||
# branch-blender-version-hash-platform: branch build
|
filename = self._strip_extension(filename)
|
||||||
filename = self._strip_extension(self.filename)
|
filename = self._strip_extension(filename)
|
||||||
|
|
||||||
tokens = filename.split("-")
|
tokens = filename.split("-")
|
||||||
|
|
||||||
if len(tokens) < 3:
|
|
||||||
return
|
|
||||||
if "blender" not in tokens:
|
|
||||||
return
|
|
||||||
|
|
||||||
blender_index = tokens.index("blender")
|
|
||||||
if not (blender_index + 2 < len(tokens)):
|
|
||||||
return
|
|
||||||
|
|
||||||
# Detect branch.
|
|
||||||
branch = "-".join(tokens[:blender_index])
|
|
||||||
|
|
||||||
# Detect version.
|
|
||||||
if branch == "":
|
|
||||||
version = tokens[blender_index + 1]
|
|
||||||
if len(tokens) == 3:
|
|
||||||
version += " release"
|
|
||||||
else:
|
|
||||||
version = ""
|
|
||||||
|
|
||||||
# Detect platform.
|
|
||||||
platform = tokens[-1]
|
|
||||||
platforms = ('osx', 'mac', 'bsd',
|
platforms = ('osx', 'mac', 'bsd',
|
||||||
'win', 'linux', 'source',
|
'win', 'linux', 'source',
|
||||||
'irix', 'solaris', 'mingw')
|
'irix', 'solaris', 'mingw')
|
||||||
platform_found = False
|
platform_tokens = []
|
||||||
for name in platforms:
|
found = False
|
||||||
if platform.lower().startswith(name):
|
|
||||||
platform_found = True
|
for i, token in enumerate(tokens):
|
||||||
if not platform_found:
|
if not found:
|
||||||
return
|
for platform in platforms:
|
||||||
|
if platform in token.lower():
|
||||||
|
found = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if found:
|
||||||
|
platform_tokens += [token]
|
||||||
|
|
||||||
|
return '-'.join(platform_tokens)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_branch(self, filename):
|
||||||
|
tokens = filename.split("-")
|
||||||
|
branch = ""
|
||||||
|
|
||||||
|
for token in tokens:
|
||||||
|
if token == "blender":
|
||||||
|
return branch
|
||||||
|
|
||||||
|
if branch == "":
|
||||||
|
branch = token
|
||||||
|
else:
|
||||||
|
branch = branch + "-" + token
|
||||||
|
|
||||||
|
return ""
|
||||||
|
|
||||||
self.version = version
|
|
||||||
self.platform = platform
|
|
||||||
self.branch = branch
|
|
||||||
self.parsed = True
|
|
||||||
|
|
||||||
def get_download_dir(branch):
|
def get_download_dir(branch):
|
||||||
|
download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
|
||||||
if not branch or branch == 'master':
|
if not branch or branch == 'master':
|
||||||
directory = DOWNLOAD_DIR
|
directory = download_prefix
|
||||||
|
elif branch == 'experimental-build':
|
||||||
|
directory = os.path.join(download_prefix, "experimental")
|
||||||
else:
|
else:
|
||||||
directory = os.path.join(DOWNLOAD_DIR, branch)
|
directory = os.path.join(download_prefix, branch)
|
||||||
os.makedirs(directory, exist_ok=True)
|
os.makedirs(directory, exist_ok=True)
|
||||||
os.chmod(directory, 0o755)
|
|
||||||
|
|
||||||
return directory
|
return directory
|
||||||
|
|
||||||
@@ -120,10 +116,6 @@ def open_zipfile(filename):
|
|||||||
def get_zipfile_packages(zipfile):
|
def get_zipfile_packages(zipfile):
|
||||||
# List packages in zip file
|
# List packages in zip file
|
||||||
packages = [Package(zipname) for zipname in zipfile.namelist()]
|
packages = [Package(zipname) for zipname in zipfile.namelist()]
|
||||||
for package in packages:
|
|
||||||
if not package.parsed:
|
|
||||||
sys.stderr.write("Failed to parse package filename: %s\n" % str(package.filename))
|
|
||||||
sys.exit(1)
|
|
||||||
if len(packages) == 0:
|
if len(packages) == 0:
|
||||||
sys.stderr.write('Empty zip file\n')
|
sys.stderr.write('Empty zip file\n')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
@@ -133,7 +125,6 @@ def get_branch_platform(packages):
|
|||||||
# Extract branch and platform names
|
# Extract branch and platform names
|
||||||
branch = packages[0].branch
|
branch = packages[0].branch
|
||||||
platform = packages[0].platform
|
platform = packages[0].platform
|
||||||
version = packages[0].version
|
|
||||||
|
|
||||||
if platform == '':
|
if platform == '':
|
||||||
sys.stderr.write('Failed to detect platform ' +
|
sys.stderr.write('Failed to detect platform ' +
|
||||||
@@ -141,11 +132,11 @@ def get_branch_platform(packages):
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
for package in packages:
|
for package in packages:
|
||||||
if package.branch != branch or package.platform != platform or package.version != version:
|
if package.branch != branch or package.platform != platform:
|
||||||
sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
|
sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
return branch, platform, version
|
return branch, platform
|
||||||
|
|
||||||
def extract_zipfile_packages(zipfile, packages, branch):
|
def extract_zipfile_packages(zipfile, packages, branch):
|
||||||
# Extract packages from zip file
|
# Extract packages from zip file
|
||||||
@@ -166,14 +157,13 @@ def extract_zipfile_packages(zipfile, packages, branch):
|
|||||||
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
|
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def remove_replaced_packages(branch, platform, version, new_packages):
|
def remove_older_packages(branch, platform, new_packages):
|
||||||
# Remove other files from the same platform and branch that are replaced
|
# Remove other files from the same platform and branch
|
||||||
# by the new packages.
|
|
||||||
directory = get_download_dir(branch)
|
directory = get_download_dir(branch)
|
||||||
|
|
||||||
for filename in os.listdir(directory):
|
for filename in os.listdir(directory):
|
||||||
package = Package(filename)
|
package = Package(filename)
|
||||||
if package.platform == platform and package.branch == branch and package.version == version:
|
if package.platform == platform and package.branch == branch:
|
||||||
is_new_package = False
|
is_new_package = False
|
||||||
for new_package in new_packages:
|
for new_package in new_packages:
|
||||||
if package.filename == new_package.filename:
|
if package.filename == new_package.filename:
|
||||||
@@ -181,32 +171,10 @@ def remove_replaced_packages(branch, platform, version, new_packages):
|
|||||||
|
|
||||||
if not is_new_package:
|
if not is_new_package:
|
||||||
try:
|
try:
|
||||||
print("Removing older package version", filename)
|
|
||||||
os.remove(os.path.join(directory, filename))
|
os.remove(os.path.join(directory, filename))
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
sys.stderr.write('Failed to remove replaced package: %s\n' % str(ex))
|
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
|
||||||
|
|
||||||
def remove_old_packages():
|
|
||||||
# Remove any branch packages that are 100 days or older.
|
|
||||||
cutoff_time = time.time() - 100 * 86400
|
|
||||||
|
|
||||||
for dirname in os.listdir(DOWNLOAD_DIR):
|
|
||||||
dirpath = os.path.join(DOWNLOAD_DIR, dirname)
|
|
||||||
if not os.path.isdir(dirpath):
|
|
||||||
continue
|
|
||||||
|
|
||||||
for filename in os.listdir(dirpath):
|
|
||||||
filepath = os.path.join(dirpath, filename)
|
|
||||||
if not os.path.isfile(filepath):
|
|
||||||
continue
|
|
||||||
|
|
||||||
file_mtime = os.stat(filepath).st_mtime
|
|
||||||
if file_mtime < cutoff_time:
|
|
||||||
try:
|
|
||||||
print("Removing old branch build", filename)
|
|
||||||
os.remove(filepath)
|
|
||||||
except Exception as ex:
|
|
||||||
sys.stderr.write('Failed to remove old package: %s\n' % str(ex))
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
@@ -215,8 +183,7 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
with open_zipfile(args.filename) as zipfile:
|
with open_zipfile(args.filename) as zipfile:
|
||||||
packages = get_zipfile_packages(zipfile)
|
packages = get_zipfile_packages(zipfile)
|
||||||
branch, platform, version = get_branch_platform(packages)
|
branch, platform = get_branch_platform(packages)
|
||||||
extract_zipfile_packages(zipfile, packages, branch)
|
extract_zipfile_packages(zipfile, packages, branch)
|
||||||
remove_replaced_packages(branch, platform, version, packages)
|
|
||||||
|
|
||||||
remove_old_packages()
|
remove_older_packages(branch, platform, packages)
|
||||||
|
Reference in New Issue
Block a user