Build bot release pipeline

* downloads builds from builder.blender.org
* generates checksums (still need to cleanup filenames)
* create source archive
* upload to download.blender.org

Next steps
* Next step is to place them in the correct folder
* file permissions download.blender.org
* snap package
* steam packages
* release notes

TODO:
* add more generic functions in master.cfg
* use a separate folder per build

build_files/buildbot-lts/README.md

@@ -0,0 +1,34 @@
+Buildbot-lts
+============
+
+This folder contains configuration files and build scripts for making a release.
+It originated when we started to do LTS releases what happens once every two
+weeks. The idea is to automate the manual steps where possible and provide a
+pipeline process for the manual steps that still needs to happen.
+
+By using the same software as builder.blender.org it would already put us in
+the right direction when we might want to make this part of builder.blender.org
+for now it is running on a VM in the blender institute as it contains user
+private keys and the process needs to be controlled security wise.
+
+But of course the source and configurations are public available for anyone to 
+check, develop and use.
+
+Setting up build-bot
+--------------------
+
+instructions from https://github.com/cjolowicz/docker-buildbot.
+
+Create custom buildbot worker containing packages we need for building (git, wget).
+
+    cd docker
+    docker build -t buildbot --no-cache .
+    cd worker
+    docker build -t buildbot-worker --no-cache .
+
+    docker network create net-buildbot
+
+    docker rm lts-buildbot && docker run --init --name lts-buildbot --network net-buildbot --publish=127.0.0.1:8010:8010 -t -i  buildbot
+
+    docker rm lts-worker && docker run --init --name lts-worker --network net-buildbot --name lts-worker -e BUILDMASTER=lts-buildbot -e WORKERNAME=lts-worker -e WORKERPASS=secret -t -i  buildbot-worker
+

build_files/buildbot-lts/docker/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:18.04
+
+RUN apt update && apt upgrade -y
+RUN apt install -y \
+    gosu \
+    wget \
+    openssh-client \
+    build-essential \
+    libffi-dev \
+    libssl-dev \
+    python3-dev \
+    python3-pip
+    
+
+ENV BUILDBOT_VERSION 2.5.0
+RUN pip3 --no-cache-dir install --upgrade pip && pip --no-cache-dir install \
+    buildbot[bundle,tls]==$BUILDBOT_VERSION \
+    buildbot-docker-swarm-worker
+
+COPY buildbot.tac /var/lib/buildbot/
+COPY docker-entrypoint.sh /usr/local/bin/
+COPY master.cfg /etc/buildbot/
+COPY bin/create_checksum.sh /var/lib/buildbot/bin/
+COPY bin/upload_file.sh /var/lib/buildbot/bin/
+COPY .ssh/id_rsa /var/lib/buildbot/.ssh/
+
+RUN adduser --home /var/lib/buildbot --disabled-password --gecos '' buildbot
+WORKDIR /var/lib/buildbot
+RUN ln -s /etc/buildbot/master.cfg
+
+VOLUME /var/lib/buildbot
+EXPOSE 8010
+EXPOSE 9989
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]

build_files/buildbot-lts/docker/bin/create_checksum.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+md5sum $1 > $2.md5
+sha256sum $1 > $2.sha256

build_files/buildbot-lts/docker/bin/upload_file.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+if [ ! -f /var/lib/buildbot/.ssh/known_hosts ]
+then
+    ssh-keyscan download.blender.org > /var/lib/buildbot/.ssh/known_hosts
+fi
+scp $1 jeroen@download.blender.org:$2

build_files/buildbot-lts/docker/buildbot.tac

@@ -0,0 +1,20 @@
+import os
+import sys
+
+from twisted.application import service
+from twisted.python.log import FileLogObserver
+from twisted.python.log import ILogObserver
+
+from buildbot.master import BuildMaster
+
+basedir = os.environ.get("BUILDBOT_BASEDIR",
+    os.path.abspath(os.path.dirname(__file__)))
+configfile = 'master.cfg'
+
+# note: this line is matched against to check that this is a buildmaster
+# directory; do not edit it.
+application = service.Application('buildmaster')
+application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
+
+m = BuildMaster(basedir, configfile, umask=None)
+m.setServiceParent(application)

build_files/buildbot-lts/docker/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '1'
+services:
+  buildbot:
+    image: buildbot/buildbot-master:master
+    env_file:
+      - db.env
+    environment:
+      - BUILDBOT_CONFIG_DIR=config
+      - BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz
+      - BUILDBOT_WORKER_PORT=9989
+      - BUILDBOT_WEB_URL=http://localhost:8010/
+      - BUILDBOT_WEB_PORT=tcp:port=8010
+    links:
+      - db
+    depends_on:
+      - db
+    ports:
+      - "8010:8010"
+  db:
+    env_file:
+      - db.env
+    image: "postgres:9.4"
+    expose:
+      - 5432
+
+  worker:
+    image: "buildbot/buildbot-worker:master"
+    environment:
+      BUILDMASTER: buildbot
+      BUILDMASTER_PORT: 9989
+      WORKERNAME: lts-worker
+      WORKERPASS: pass
+      WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST
+
+    links:
+      - buildbot

build_files/buildbot-lts/docker/docker-entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+buildbot upgrade-master .
+
+chown buildbot /var/lib/buildbot/
+chown buildbot /var/lib/buildbot/.ssh/
+chown -R buildbot /var/lib/buildbot/*
+
+if [ -S /var/run/docker.sock ]
+then
+    group=$(stat -c '%g' /var/run/docker.sock)
+else
+    group=buildbot
+fi
+
+gosu buildbot:$group "$@"

build_files/buildbot-lts/docker/master.cfg

@@ -0,0 +1,240 @@
+# -*- python -*-
+# ex: set filetype=python:
+
+import os
+
+from buildbot.plugins import *
+
+# This is a sample buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory.
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+####### WORKERS
+
+# The 'workers' list defines the set of recognized workers. Each element is
+# a Worker object, specifying a unique worker name and password.  The same
+# worker name and password must be configured on the worker.
+
+c['workers'] = [worker.Worker("lts-worker", 'secret')]
+
+if 'BUILDBOT_MQ_URL' in os.environ:
+    c['mq'] = {
+        'type' : 'wamp',
+        'router_url': os.environ['BUILDBOT_MQ_URL'],
+        'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
+        'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+        'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+        'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
+    }
+# 'protocols' contains information about protocols which master will use for
+# communicating with workers. You must define at least 'port' option that workers
+# could connect to your master with this protocol.
+# 'port' must match the value configured into the workers (with their
+# --master option)
+c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
+
+####### CHANGESOURCES
+
+# the 'change_source' setting tells the buildmaster how it should find out
+# about source code changes.  We don't build when sources change.
+c['change_source'] = []
+
+####### SCHEDULERS
+
+# Configure the Schedulers, which decide how to react to incoming changes.  In this
+# case, just kick off a 'runtests' build
+
+c['schedulers'] = [
+    schedulers.ForceScheduler(
+        name="build",
+        builderNames=["release-blender-283"],
+        properties=[
+            util.StringParameter(
+                name="blender_version",
+                label="Version of blender.",
+                default="2.83"),
+            util.StringParameter(
+                name="blender_version_full",
+                label="Version of blender (Full).",
+                default="2.83.1")
+        ]
+    ),
+
+    schedulers.Triggerable(
+        name="deploy-source-archive",
+        builderNames=["deploy-source-archive"]
+    ),
+
+    schedulers.Triggerable(
+        name="deploy-buildbot-packages",
+        builderNames=["deploy-buildbot-packages"]
+    ),
+]
+
+####### BUILDERS
+
+# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+# what steps, and which workers can execute them.  Note that any particular build will
+# only take place on one worker.
+
+deploy_source_archive_factory = util.BuildFactory()
+deploy_source_archive_factory.addStep(steps.Git(name="Checkout Blender Source", repourl='https://git.blender.org/blender.git', branch=util.Interpolate('blender-v%(prop:blender_version)s-release'), mode='full'))
+deploy_source_archive_factory.addStep(steps.ShellSequence(name="Make source archive", commands=[
+    util.ShellArg(command=["make", "source_archive"], logfile='make_source_archive'),
+    # The next two shell commands are only needed during development
+    util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz')]),
+    util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz.md5sum", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum')]),
+]))
+deploy_source_archive_factory.addStep(steps.FileUpload(
+    name="Upload source archive",
+    workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz'),
+    masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz')))
+deploy_source_archive_factory.addStep(steps.FileUpload(
+    name="Upload source archive checksum",
+    workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum'),
+    masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz.md5sum')))
+deploy_source_archive_factory.addStep(steps.MasterShellCommand(
+    name="Upload source archive",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+deploy_source_archive_factory.addStep(steps.MasterShellCommand(
+    name="Upload windows64.zip build",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz.md5sum'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+
+deploy_buildbot_packages_factory = util.BuildFactory()
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Download linux64 build",
+    command=[
+        'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-linux64.tar.xz'),
+        "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-linux64.tar.xz')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Download windows64.msi build",
+    command=[
+        'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-windows64.msi'),
+        "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.msi')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Download windows64.zip build",
+    command=[
+        'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-windows64.zip'),
+        "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.zip')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Download macOS build",
+    command=[
+        'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-macOS.dmg'),
+        "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-macOS.dmg')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Create checksum (md5/sha256)",
+    command=[
+        'sh',
+        '-C',
+        '/var/lib/buildbot/bin/create_checksum.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-*'),
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s'),
+]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Upload linux64 build",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-linux64.tar.xz'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Upload windows64.msi build",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.msi'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Upload windows64.zip build",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.zip'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Upload macOS build",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-macOS.dmg'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Upload md5 checksum",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s.md5'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand(
+    name="Upload sha256 checksum",
+    command=[
+        'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh',
+        util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s.sha256'), 
+        util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')]))
+
+factory = util.BuildFactory()
+factory.addStep(steps.MasterShellCommand(name="Init release folder", command=["mkdir", "-p", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s')]))
+factory.addStep(steps.Trigger(
+    name='Deploy',
+    schedulerNames=[
+        'deploy-source-archive',
+        'deploy-buildbot-packages',
+    ],
+    waitForFinish=True,
+    set_properties={
+        'blender_version_full': util.Property('blender_version_full'),
+        'blender_version': util.Property('blender_version'),
+    }
+))
+
+# Download builds from builder.blender.org
+c['builders'] = [
+    util.BuilderConfig(name="release-blender-283",
+      workernames=["lts-worker"],
+      factory=factory),
+    util.BuilderConfig(name="deploy-source-archive",
+      workernames=["lts-worker"],
+      factory=deploy_source_archive_factory),
+    util.BuilderConfig(name="deploy-buildbot-packages",
+      workernames=["lts-worker"],
+      factory=deploy_buildbot_packages_factory),
+]
+
+####### STATUS TARGETS
+
+# 'status' is a list of Status Targets. The results of each build will be
+# pushed to these targets. buildbot/status/*.py has a variety to choose from,
+# like IRC bots.
+
+# c['status'] = []
+
+####### PROJECT IDENTITY
+
+# the 'title' string will appear at the top of this buildbot installation's
+# home pages (linked to the 'titleURL').
+
+c['title'] = "Blender Release LTS"
+c['titleURL'] = "https://www.blender.org/download/lts/"
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server is visible. This typically uses the port number set in
+# the 'www' entry below, but with an externally-visible host name which the
+# buildbot cannot figure out without some help.
+
+c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
+
+# minimalistic config to activate new web UI
+c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
+                plugins=dict(waterfall_view={}, console_view={}))
+
+####### DB URL
+
+c['db'] = {
+    # This specifies what database buildbot uses to store its state.  You can leave
+    # this at its default for all but the largest installations.
+    'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
+}

build_files/buildbot-lts/docker/worker/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:18.04
+
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    git \
+    wget \
+    python3-dev \
+    python3-pip \
+    python3-setuptools \
+    python3-wheel \
+    ; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV BUILDBOT_VERSION 2.5.0
+RUN pip3 install --upgrade pip && \
+    pip --no-cache-dir install twisted[tls] && \
+    pip --no-cache-dir install buildbot_worker==$BUILDBOT_VERSION
+
+RUN useradd --create-home --home-dir=/var/lib/buildbot buildbot
+WORKDIR /var/lib/buildbot
+USER buildbot
+
+COPY buildbot.tac .
+
+CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]

build_files/buildbot-lts/docker/worker/buildbot.tac

@@ -0,0 +1,39 @@
+import fnmatch
+import os
+import sys
+
+from twisted.application import service
+from twisted.python.log import FileLogObserver
+from twisted.python.log import ILogObserver
+
+from buildbot_worker.bot import Worker
+
+# setup worker
+basedir = os.path.abspath(os.path.dirname(__file__))
+application = service.Application('buildbot-worker')
+
+
+application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
+# and worker on the same process!
+buildmaster_host = os.environ.get("BUILDMASTER", 'localhost')
+port = int(os.environ.get("BUILDMASTER_PORT", 9989))
+workername = os.environ.get("WORKERNAME", 'docker')
+passwd = os.environ.get("WORKERPASS")
+
+# delete the password from the environ so that it is not leaked in the log
+blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split()
+for name in list(os.environ.keys()):
+    for toremove in blacklist:
+        if fnmatch.fnmatch(name, toremove):
+            del os.environ[name]
+
+keepalive = 600
+umask = None
+maxdelay = 300
+allow_shutdown = None
+maxretries = 10
+
+s = Worker(buildmaster_host, port, workername, passwd, basedir,
+           keepalive, umask=umask, maxdelay=maxdelay,
+           allow_shutdown=allow_shutdown, maxRetries=maxretries)
+s.setServiceParent(application)