1
1

Compare commits

...

2 Commits

Author SHA1 Message Date
2cbc328bdc Build bot release pipeline
* downloads builds from builder.blender.org
* generates checksums (still need to cleanup filenames)
* create source archive
* upload to download.blender.org

Next steps
* Next step is to place them in the correct folder
* file permissions download.blender.org
* snap package
* steam packages
* release notes

TODO:
* add more generic functions in master.cfg
* use a separate folder per build
2020-07-03 16:19:50 +02:00
314783f2e7 BuildbotLTS: Project Documentation
Added README.md containing a overview of the project.
2020-07-02 16:56:26 +02:00
10 changed files with 461 additions and 0 deletions

View File

@@ -0,0 +1,34 @@
Buildbot-lts
============
This folder contains configuration files and build scripts for making a release.
It originated when we started to do LTS releases what happens once every two
weeks. The idea is to automate the manual steps where possible and provide a
pipeline process for the manual steps that still needs to happen.
By using the same software as builder.blender.org it would already put us in
the right direction when we might want to make this part of builder.blender.org
for now it is running on a VM in the blender institute as it contains user
private keys and the process needs to be controlled security wise.
But of course the source and configurations are public available for anyone to
check, develop and use.
Setting up build-bot
--------------------
instructions from https://github.com/cjolowicz/docker-buildbot.
Create custom buildbot worker containing packages we need for building (git, wget).
cd docker
docker build -t buildbot --no-cache .
cd worker
docker build -t buildbot-worker --no-cache .
docker network create net-buildbot
docker rm lts-buildbot && docker run --init --name lts-buildbot --network net-buildbot --publish=127.0.0.1:8010:8010 -t -i buildbot
docker rm lts-worker && docker run --init --name lts-worker --network net-buildbot --name lts-worker -e BUILDMASTER=lts-buildbot -e WORKERNAME=lts-worker -e WORKERPASS=secret -t -i buildbot-worker

View File

@@ -0,0 +1,36 @@
FROM ubuntu:18.04
RUN apt update && apt upgrade -y
RUN apt install -y \
gosu \
wget \
openssh-client \
build-essential \
libffi-dev \
libssl-dev \
python3-dev \
python3-pip
ENV BUILDBOT_VERSION 2.5.0
RUN pip3 --no-cache-dir install --upgrade pip && pip --no-cache-dir install \
buildbot[bundle,tls]==$BUILDBOT_VERSION \
buildbot-docker-swarm-worker
COPY buildbot.tac /var/lib/buildbot/
COPY docker-entrypoint.sh /usr/local/bin/
COPY master.cfg /etc/buildbot/
COPY bin/create_checksum.sh /var/lib/buildbot/bin/
COPY bin/upload_file.sh /var/lib/buildbot/bin/
COPY .ssh/id_rsa /var/lib/buildbot/.ssh/
RUN adduser --home /var/lib/buildbot --disabled-password --gecos '' buildbot
WORKDIR /var/lib/buildbot
RUN ln -s /etc/buildbot/master.cfg
VOLUME /var/lib/buildbot
EXPOSE 8010
EXPOSE 9989
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]

View File

@@ -0,0 +1,4 @@
#!/bin/sh
set -e
md5sum $1 > $2.md5
sha256sum $1 > $2.sha256

View File

@@ -0,0 +1,7 @@
#!/bin/sh
set -e
if [ ! -f /var/lib/buildbot/.ssh/known_hosts ]
then
ssh-keyscan download.blender.org > /var/lib/buildbot/.ssh/known_hosts
fi
scp $1 jeroen@download.blender.org:$2

View File

@@ -0,0 +1,20 @@
import os
import sys
from twisted.application import service
from twisted.python.log import FileLogObserver
from twisted.python.log import ILogObserver
from buildbot.master import BuildMaster
basedir = os.environ.get("BUILDBOT_BASEDIR",
os.path.abspath(os.path.dirname(__file__)))
configfile = 'master.cfg'
# note: this line is matched against to check that this is a buildmaster
# directory; do not edit it.
application = service.Application('buildmaster')
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
m = BuildMaster(basedir, configfile, umask=None)
m.setServiceParent(application)

View File

@@ -0,0 +1,36 @@
version: '1'
services:
buildbot:
image: buildbot/buildbot-master:master
env_file:
- db.env
environment:
- BUILDBOT_CONFIG_DIR=config
- BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
- BUILDBOT_WORKER_PORT=9989
- BUILDBOT_WEB_URL=http://localhost:8010/
- BUILDBOT_WEB_PORT=tcp:port=8010
links:
- db
depends_on:
- db
ports:
- "8010:8010"
db:
env_file:
- db.env
image: "postgres:9.4"
expose:
- 5432
worker:
image: "buildbot/buildbot-worker:master"
environment:
BUILDMASTER: buildbot
BUILDMASTER_PORT: 9989
WORKERNAME: lts-worker
WORKERPASS: pass
WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
links:
- buildbot

View File

@@ -0,0 +1,18 @@
#!/bin/sh
set -e
buildbot upgrade-master .
chown buildbot /var/lib/buildbot/
chown buildbot /var/lib/buildbot/.ssh/
chown -R buildbot /var/lib/buildbot/*
if [ -S /var/run/docker.sock ]
then
group=$(stat -c '%g' /var/run/docker.sock)
else
group=buildbot
fi
gosu buildbot:$group "$@"

View File

@@ -0,0 +1,240 @@
# -*- python -*-
# ex: set filetype=python:
import os
from buildbot.plugins import *
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
####### WORKERS
# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password. The same
# worker name and password must be configured on the worker.
c['workers'] = [worker.Worker("lts-worker", 'secret')]
if 'BUILDBOT_MQ_URL' in os.environ:
c['mq'] = {
'type' : 'wamp',
'router_url': os.environ['BUILDBOT_MQ_URL'],
'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
}
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. We don't build when sources change.
c['change_source'] = []
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
c['schedulers'] = [
schedulers.ForceScheduler(
name="build",
builderNames=["release-blender-283"],
properties=[
util.StringParameter(
name="blender_version",
label="Version of blender.",
default="2.83"),
util.StringParameter(
name="blender_version_full",
label="Version of blender (Full).",
default="2.83.1")
]
),
schedulers.Triggerable(
name="deploy-source-archive",
builderNames=["deploy-source-archive"]
),
schedulers.Triggerable(
name="deploy-buildbot-packages",
builderNames=["deploy-buildbot-packages"]
),
]
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
deploy_source_archive_factory = util.BuildFactory()
deploy_source_archive_factory.addStep(steps.Git(name="Checkout Blender Source", repourl='https://git.blender.org/blender.git', branch=util.Interpolate('blender-v%(prop:blender_version)s-release'), mode='full'))
deploy_source_archive_factory.addStep(steps.ShellSequence(name="Make source archive", commands=[
util.ShellArg(command=["make", "source_archive"], logfile='make_source_archive'),
# The next two shell commands are only needed during development
util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz')]),
util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz.md5sum", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum')]),
]))
deploy_source_archive_factory.addStep(steps.FileUpload(
name="Upload source archive",
workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz'),
masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz')))
deploy_source_archive_factory.addStep(steps.FileUpload(
name="Upload source archive checksum",
workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum'),
masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz.md5sum')))
deploy_source_archive_factory.addStep(steps.MasterShellCommand(
name="Upload source archive",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_source_archive_factory.addStep(steps.MasterShellCommand(
name="Upload windows64.zip build",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz.md5sum'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_buildbot_packages_factory = util.BuildFactory()
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Download linux64 build",
command=[
'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-linux64.tar.xz'),
"-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-linux64.tar.xz')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Download windows64.msi build",
command=[
'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-windows64.msi'),
"-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.msi')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Download windows64.zip build",
command=[
'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-windows64.zip'),
"-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.zip')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Download macOS build",
command=[
'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-macOS.dmg'),
"-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-macOS.dmg')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Create checksum (md5/sha256)",
command=[
'sh',
'-C',
'/var/lib/buildbot/bin/create_checksum.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-*'),
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s'),
]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Upload linux64 build",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-linux64.tar.xz'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Upload windows64.msi build",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.msi'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Upload windows64.zip build",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.zip'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Upload macOS build",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-macOS.dmg'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Upload md5 checksum",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s.md5'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
name="Upload sha256 checksum",
command=[
'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s.sha256'),
util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
factory = util.BuildFactory()
factory.addStep(steps.MasterShellCommand(name="Init release folder", command=["mkdir", "-p", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s')]))
factory.addStep(steps.Trigger(
name='Deploy',
schedulerNames=[
'deploy-source-archive',
'deploy-buildbot-packages',
],
waitForFinish=True,
set_properties={
'blender_version_full': util.Property('blender_version_full'),
'blender_version': util.Property('blender_version'),
}
))
# Download builds from builder.blender.org
c['builders'] = [
util.BuilderConfig(name="release-blender-283",
workernames=["lts-worker"],
factory=factory),
util.BuilderConfig(name="deploy-source-archive",
workernames=["lts-worker"],
factory=deploy_source_archive_factory),
util.BuilderConfig(name="deploy-buildbot-packages",
workernames=["lts-worker"],
factory=deploy_buildbot_packages_factory),
]
####### STATUS TARGETS
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# like IRC bots.
# c['status'] = []
####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c['title'] = "Blender Release LTS"
c['titleURL'] = "https://www.blender.org/download/lts/"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
# minimalistic config to activate new web UI
c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
plugins=dict(waterfall_view={}, console_view={}))
####### DB URL
c['db'] = {
# This specifies what database buildbot uses to store its state. You can leave
# this at its default for all but the largest installations.
'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
}

View File

@@ -0,0 +1,27 @@
FROM ubuntu:18.04
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends \
build-essential \
git \
wget \
python3-dev \
python3-pip \
python3-setuptools \
python3-wheel \
; \
rm -rf /var/lib/apt/lists/*
ENV BUILDBOT_VERSION 2.5.0
RUN pip3 install --upgrade pip && \
pip --no-cache-dir install twisted[tls] && \
pip --no-cache-dir install buildbot_worker==$BUILDBOT_VERSION
RUN useradd --create-home --home-dir=/var/lib/buildbot buildbot
WORKDIR /var/lib/buildbot
USER buildbot
COPY buildbot.tac .
CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]

View File

@@ -0,0 +1,39 @@
import fnmatch
import os
import sys
from twisted.application import service
from twisted.python.log import FileLogObserver
from twisted.python.log import ILogObserver
from buildbot_worker.bot import Worker
# setup worker
basedir = os.path.abspath(os.path.dirname(__file__))
application = service.Application('buildbot-worker')
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
# and worker on the same process!
buildmaster_host = os.environ.get("BUILDMASTER", 'localhost')
port = int(os.environ.get("BUILDMASTER_PORT", 9989))
workername = os.environ.get("WORKERNAME", 'docker')
passwd = os.environ.get("WORKERPASS")
# delete the password from the environ so that it is not leaked in the log
blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split()
for name in list(os.environ.keys()):
for toremove in blacklist:
if fnmatch.fnmatch(name, toremove):
del os.environ[name]
keepalive = 600
umask = None
maxdelay = 300
allow_shutdown = None
maxretries = 10
s = Worker(buildmaster_host, port, workername, passwd, basedir,
keepalive, umask=umask, maxdelay=maxdelay,
allow_shutdown=allow_shutdown, maxRetries=maxretries)
s.setServiceParent(application)