Remove 291

Explicit setting of nighlty build usage
Disable most nightly jobs

.arcconfig

@@ -0,0 +1,8 @@
+{
+	"project_id" : "Infrastructure: Blender Buildbot",
+	"conduit_uri" : "https://developer.blender.org/",
+	"phabricator.uri" : "https://developer.blender.org/",
+	"git.default-relative-commit" : "origin/master",
+	"arc.land.update.default" : "rebase",
+	"arc.land.onto.default" : "master"
+}

master.cfg

@@ -1,59 +1,92 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
 # -*- python -*-
 # ex: set syntax=python:
 
 # <pep8 compliant>
 
-# List of the branches being built automatically overnight
-NIGHT_SCHEDULE_BRANCHES = ["master"]
 
-# List of the branches available for force build
+from buildbot.www.authz.roles import RolesFromBase
-FORCE_SCHEDULE_BRANCHES = ["master", "experimental-build"]
+from buildbot.www.authz.roles import RolesFromOwner
+from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.transfer import FileUpload
+from buildbot.steps.shell import Test
+from buildbot.steps.shell import Compile
+from buildbot.steps.shell import ShellCommand
+from buildbot.process.properties import Interpolate
+from buildbot.process.factory import BuildFactory
+from buildbot.plugins import steps, util
+from buildbot.config import BuilderConfig
+from buildbot.schedulers import timed, forcesched
+from buildbot.schedulers.basic import SingleBranchScheduler
+from buildbot.schedulers.filter import ChangeFilter
+from buildbot.changes.gitpoller import GitPoller
+from buildbot.changes.svnpoller import SVNPoller
+from buildbot.worker import Worker
 
-# List of branches availble for addons branch selection.
+from datetime import timedelta
-ADDONS_BRANCHES = ["master"]
 
-# List of branches availble for libraries branch selection.
+# NOTE: We load the workers and their passwords from a separator file, so we can
-LIB_BRANCHES = ["trunk"]
+# have # this one in Git.
+import master_private
 
 # Dictionary that the buildmaster pays attention to.
 c = BuildmasterConfig = {}
 
-# BUILD WORKERS
+# Project identity.
-#
+c['projectName'] = 'Blender'
-# We load the slaves and their passwords from a separator file, so we can have
+c['projectURL'] = 'https://www.blender.org/'
-# this one in SVN.
+c['title'] = 'Blender'
+c['titleURL'] = 'https://builder.blender.org/'
 
-from buildbot.worker import Worker
+# Buildbot information.
-import master_private
+c['buildbotURL'] = 'https://builder.blender.org/admin/'
+c['buildbotNetUsageData'] = 'basic'
+
+# Various.
+c['db_url'] = 'sqlite:///state.sqlite'
+
+# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
+c['buildbotNetUsageData'] = None
+
+################################################################################
+# BUILD WORKERS
 
 c['workers'] = []
-for slave in master_private.slaves:
+for worker in master_private.workers:
-    c['workers'].append(Worker(slave['name'], slave['password']))
+    c['workers'].append(
+        Worker(worker['name'], worker['password'], max_builds=1))
 
-# TCP port through which slaves connect
+# TCP port through which workers connect
 c['protocols'] = {
-    "pb": {
+    'pb': {
-        "port": "tcp:{}".format(9989)
+        'port': 'tcp:{}'.format(9989)
     }
 }
-# CHANGE SOURCES
-
-from buildbot.changes.svnpoller import SVNPoller
-from buildbot.changes.gitpoller import GitPoller
-
-c['change_source'] = GitPoller('git://git.blender.org/blender.git',
-                               pollinterval=1200)
 
+################################################################################
 # CODEBASES
 #
 # Allow to control separately things like branches for each repo and submodules.
 
 all_repositories = {
     r'git://git.blender.org/blender.git': 'blender',
-    r'git://git.blender.org/blender-translations.git': 'blender-translations',
-    r'git://git.blender.org/blender-addons.git': 'blender-addons',
-    r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
-    r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
 }
 
@@ -65,179 +98,130 @@ def codebaseGenerator(chdict):
 c['codebaseGenerator'] = codebaseGenerator
 
 
+################################################################################
 # SCHEDULERS
 #
 # Decide how to react to incoming changes.
 
-from buildbot.schedulers import timed, forcesched
 
 c['schedulers'] = []
 
 
-def schedule_force_build(name):
+def schedule_force_build(name, branch):
     """
     Makes it possible to have "Force Build" for the given builder.
-    Makes sure only reasonabel subset of properties are exposed.
+    Makes sure only reasonable subset of properties are exposed.
     """
+    if branch != '':
+        branch_parameter = forcesched.FixedParameter(
+            name='branch',
+            default=branch,
+            hide=True)
+    else:
+        branch_parameter = forcesched.StringParameter(
+            name='branch',
+            label='Branch:',
+            default='custom-branch-name-here',
+            regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$')
+
     c['schedulers'].append(forcesched.ForceScheduler(
         name='force_' + name,
-        buttonName="Force Build",
+        buttonName='Force Build',
         builderNames=[name],
         codebases=[forcesched.CodebaseParameter(
-                codebase="blender",
+            codebase='blender',
-                branch=forcesched.ChoiceStringParameter(
+            branch=branch_parameter,
-                    name="branch",
+            # Hide revision. We don't want to allow anyone to overwrite the
-                    choices=FORCE_SCHEDULE_BRANCHES,
+            # master build with an older version. Could be added back once we
-                    default="master"),
+            # have authentication.
-                # Do not hide revision, can be handy!
+            revision=forcesched.FixedParameter(
-                repository=forcesched.FixedParameter(
+                name='revision',
-                    name="repository",
+                default='',
-                    default="",
+                hide=True),
-                    hide=True),
+            repository=forcesched.FixedParameter(
-                project=forcesched.FixedParameter(
+                name='repository',
-                    name="project",
+                default='',
-                    default="",
+                hide=True),
-                    hide=True)),
+            project=forcesched.FixedParameter(
-                   # For now, hide other codebases.
+                name='project',
-                   forcesched.CodebaseParameter(
+                default='',
-                       hide=True,
+                hide=True)),
-                       codebase="blender-translations"),
+        ]))
-                   forcesched.CodebaseParameter(
-                       codebase="blender-addons",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=ADDONS_BRANCHES,
-                           default="master"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-                   forcesched.CodebaseParameter(
-                       codebase="blender-addons-contrib",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=ADDONS_BRANCHES,
-                           default="master"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-                   forcesched.CodebaseParameter(
-                       hide=True,
-                       codebase="blender-dev-tools"),
-                   forcesched.CodebaseParameter(
-                       codebase="lib svn",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=LIB_BRANCHES,
-                           default="trunk"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-        ],
-        properties=[]))
 
 
-def schedule_nightly_build(name, hour, minute=0):
+def schedule_nightly_build(name, branch, hour, minute=0):
     """
     Creates scheduler for nightly builds for a given builder.
     """
-    for current_branch in NIGHT_SCHEDULE_BRANCHES:
+    scheduler_name = f'nightly_{name} {branch}'
-        scheduler_name = "nightly_" + name
+    c['schedulers'].append(timed.Nightly(
-        if current_branch:
+        name=scheduler_name,
-            scheduler_name += ' ' + current_branch
+        codebases={
-        addons_branch = "master"
+            'blender': {'repository': '',
-        c['schedulers'].append(timed.Nightly(
+                        'branch': branch}},
-            name=scheduler_name,
+        branch=branch,
-            codebases={
+        builderNames=[name],
-                "blender": {"repository": "",
+        hour=hour,
-                            "branch": current_branch},
+        minute=minute))
-                "blender-translations": {"repository": "",
-                                         "branch": "master"},
-                "blender-addons": {"repository": "",
-                                   "branch": addons_branch},
-                "blender-addons-contrib": {"repository": "",
-                                           "branch": addons_branch},
-                "blender-dev-tools": {"repository": "",
-                                      "branch": "master"},
-                "lib svn": {"repository": "",
-                            "branch": "trunk"}},
-            branch=current_branch,
-            builderNames=[name],
-            hour=hour,
-            minute=minute))
 
 
+def schedule_change_build(name, branch):
+    """
+    Creates scheduler for building on changes.
+    This will not package and upload the build.
+    """
+    scheduler_name = f'change_{name} {branch}'
+    c['schedulers'].append(SingleBranchScheduler(
+        name=scheduler_name,
+        codebases={
+            'blender': {'repository': '',
+                        'branch': branch}},
+        builderNames=[name],
+        treeStableTimer=120,
+        change_filter=ChangeFilter(project=['blender'], branch=branch),
+        properties={'skip_upload': True, 'skip_codesign': True}))
+
+
+################################################################################
 # BUILDERS
 #
 # The 'builders' list defines the Builders, which tell Buildbot how to
-# perform a build: what steps, and which slaves can execute them.
+# perform a build: what steps, and which workers can execute them.
-# Note that any particular build will only take place on one slave.
+# Note that any particular build will only take place on one worker.
-
-from buildbot.config import BuilderConfig
-from buildbot.plugins import steps, util
-from buildbot.process.factory import BuildFactory
-from buildbot.process.properties import Interpolate
-from buildbot.steps.shell import ShellCommand
-from buildbot.steps.shell import Compile
-from buildbot.steps.shell import Test
-from buildbot.steps.transfer import FileUpload
-from buildbot.steps.master import MasterShellCommand
-
-# add builder utility
 
 c['builders'] = []
-buildernames = []
+builders_all_branches = set()
 
 
-def add_builder(c, name, libdir, factory, branch='',
+# Add builder utility.
-                rsync=False, hour=3, minute=0):
-    workernames = []
 
-    for slave in master_private.slaves:
+def add_builder(c, name, platforms, factory, branch='', hour=3, minute=0, use_nightly_builder=True):
-        if name in slave['builders']:
+    if branch != '':
-            workernames.append(slave['name'])
+        builders_all_branches.add(branch)
 
-    if workernames:
+    for platform in platforms:
-        f = factory(name, libdir, branch, rsync)
+        workernames = []
-        c['builders'].append(BuilderConfig(name=name,
+        builder_name = f'{name}_{platform}'
-                                           workernames=workernames,
-                                           factory=f,
-                                           tags=['blender']))
-        buildernames.append(name)
 
-        schedule_nightly_build(name, hour, minute)
+        for worker in master_private.workers:
-        schedule_force_build(name)
+            if platform == worker['platform']:
+                workernames.append(worker['name'])
+                builder_name = f"{worker['platform_short']}_{name}"
+
+        if workernames:
+            f = factory(builder_name, branch)
+            c['builders'].append(BuilderConfig(name=builder_name,
+                                               workernames=workernames,
+                                               factory=f,
+                                               tags=['blender']))
+
+            if use_nightly_builder:
+                schedule_nightly_build(builder_name, branch, hour, minute)
+                schedule_change_build(builder_name, branch)
+            schedule_force_build(builder_name, branch)
 
 
-# common steps
+# Common steps.
-
-def git_submodule_step(submodule):
-    return steps.Git(name=submodule + '.git',
-                     repourl='git://git.blender.org/' + submodule + '.git',
-                     mode='incremental',
-                     codebase=submodule,
-                     workdir=submodule + '.git')
-
 
 def git_step(branch=''):
     if branch:
@@ -257,96 +241,118 @@ def git_step(branch=''):
                          submodules=True)
 
 
-def git_submodules_update():
+# Generic builder.
-    command = ['git', 'submodule', 'update', '--remote']
-    return ShellCommand(name='Submodules Update',
-                        command=command,
-                        description='updating',
-                        descriptionDone='up to date',
-                        workdir='blender.git')
 
+def do_upload(step):
+    return not step.hasProperty('skip_upload')
 
-def lib_svn_step(dir):
+@util.renderer
-    # TODO(sergey): For some reason interpolation is always giving empty branch.
+def script_command(props, script, id, branch):
-    # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
+    # NOTE: On Windows never includes major version in the executable name,
-    lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
+    # so Python 3 will have be 'python.exe'.
-    return steps.SVN(name='lib svn ' + dir,
+    if id.startswith('win'):
-                     repourl=util.Interpolate(lib_repo + dir),
+        python_command = 'python'
-                     codebase='lib svn',
+    else:
-                     mode='incremental',
+        python_command = 'python3'
-                     workdir='lib/' + dir)
 
+    git_branch = branch or Interpolate('%(src:blender:branch)s')
+    args = [python_command, script, id, git_branch]
 
-def rsync_step(id, branch, rsync_script):
+    if not props.hasProperty('skip_codesign'):
-    return ShellCommand(name='rsync',
+        args += ['--codesign']
-                        command=['python3', rsync_script, id, branch],
-                        description='uploading',
-                        descriptionDone='uploaded',
-                        workdir='install')
 
+    return args
 
-# generic builder
+def generic_builder(id, branch=''):
-
+    # TODO(sergey): Consider using pathlib.
-def generic_builder(id, libdir='', branch='', rsync=False):
+    filename = f'uploaded/buildbot_upload_{id}.zip'
-    filename = 'uploaded/buildbot_upload_' + id + '.zip'
+    update_script = '../blender.git/build_files/buildbot/worker_update.py'
-    compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
+    compile_script = '../blender.git/build_files/buildbot/worker_compile.py'
-    test_script = '../blender.git/build_files/buildbot/slave_test.py'
+    test_script = '../blender.git/build_files/buildbot/worker_test.py'
-    pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
+    pack_script = '../blender.git/build_files/buildbot/worker_pack.py'
-    rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
     unpack_script = 'master_unpack.py'
 
     f = BuildFactory()
-    if libdir != '':
-        f.addStep(lib_svn_step(libdir))
-
-    f.addStep(lib_svn_step('tests'))
-
-    for submodule in ('blender-translations',
-                      'blender-addons',
-                      'blender-addons-contrib',
-                      'blender-dev-tools'):
-        f.addStep(git_submodule_step(submodule))
-
     f.addStep(git_step(branch))
-    f.addStep(git_submodules_update())
 
-    f.addStep(Compile(command=['python3', compile_script, id], timeout=3600))
+    f.addStep(ShellCommand(
-    f.addStep(Test(command=['python3', test_script, id]))
+        name='submodules and libraries update',
+        command=script_command.withArgs(update_script, id, branch),
+        description='updating',
+        descriptionDone='updated',
+        haltOnFailure=True))
+
+    f.addStep(Compile(
+        command=script_command.withArgs(compile_script, id, branch),
+        timeout=3600))
+    f.addStep(Test(
+        command=script_command.withArgs(test_script, id, branch)))
     f.addStep(ShellCommand(
         name='package',
-        command=['python3',
+        command=script_command.withArgs(pack_script, id, branch),
-                 pack_script,
-                 id,
-                 branch or Interpolate('%(src:blender:branch)s')],
         description='packaging',
-        descriptionDone='packaged'))
+        descriptionDone='packaged',
-    if rsync:
+        haltOnFailure=True,
-        f.addStep(rsync_step(id, branch, rsync_script))
+        doStepIf=do_upload))
-    else:
+
-        f.addStep(FileUpload(name='upload',
+    f.addStep(FileUpload(name='upload',
-                             workersrc='buildbot_upload.zip',
+                         workersrc='buildbot_upload.zip',
-                             masterdest=filename,
+                         masterdest=filename,
-                             maxsize=180 * 1024 * 1024,
+                         maxsize=500 * 1024 * 1024,
-                             workdir='install'))
+                         workdir='install',
+                         doStepIf=do_upload))
     f.addStep(MasterShellCommand(name='unpack',
-                                 command=['python3.6',
+                                 command=['python3.7',
                                           unpack_script,
                                           filename],
                                  description='unpacking',
-                                 descriptionDone='unpacked'))
+                                 descriptionDone='unpacked',
+                                 doStepIf=do_upload))
     return f
 
 
+################################################################################
 # Builders
 
-add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
+add_builder(c,
-add_builder(c, 'linux_glibc217_x86_64_cmake', '', generic_builder, hour=1)
+            'master',
-# NOTE: Visual Studio 2017 (vc15) is using libraries folder from
+            ['windows', 'macOS_10_15', 'linux_centos7'],
-# Visual Studio 2015 (vc14)
+            generic_builder,
-add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=1)
+            branch='master',
+            hour=1,
+            use_nightly_builder=True)
+add_builder(c,
+            '292',
+            ['windows', 'macOS_10_15', 'linux_centos7'],
+            generic_builder,
+            branch='blender-v2.92-release',
+            hour=1,
+            use_nightly_builder=False)
+add_builder(c,
+            'lts_283',
+            ['windows', 'macOS_10_13', 'linux_centos7'],
+            generic_builder,
+            branch='blender-v2.83-release',
+            hour=1,
+            use_nightly_builder=False)
+add_builder(c,
+            'custom_branch',
+            ['windows', 'macOS_10_15', 'linux_centos7'],
+            generic_builder,
+            branch='',
+            hour=1,
+            use_nightly_builder=False)
 
+################################################################################
+# CHANGE SOURCES
+
+c['change_source'] = GitPoller(repourl='git://git.blender.org/blender.git',
+                               pollinterval=120,
+                               project='blender',
+                               branches=list(builders_all_branches))
+
+################################################################################
 # HORIZONS
-from datetime import timedelta
 
 c['changeHorizon'] = 300
 
@@ -358,15 +364,17 @@ c['configurators'] = [util.JanitorConfigurator(
     dayOfWeek=6)]
 
 
+################################################################################
 # WWW
+
 c['www'] = dict(port=8010,
                 plugins={'console_view': {},
                          'grid_view': {},
                          'waterfall_view': {}})
 
+################################################################################
 # Access
-from buildbot.www.authz.roles import RolesFromOwner
+
-from buildbot.www.authz.roles import RolesFromBase
 
 class StrangerRoles(RolesFromBase):
     def getRolesFromUser(self, userDetails):
@@ -374,42 +382,31 @@ class StrangerRoles(RolesFromBase):
 
 
 authz = util.Authz(
-    stringsMatcher=util.fnmatchStrMatcher,  # simple matcher with '*' glob character
+    # Simple matcher with '*' glob character.
-    # stringsMatcher = util.reStrMatcher,   # if you prefer regular expressions
+    # Could be util.reStrMatcher if regular expressions are preferred.
+    stringsMatcher=util.fnmatchStrMatcher,
+
     allowRules=[
-        # admins can do anything,
+        # Admins can do anything,
-        # defaultDeny=False: if user does not have the admin role, we continue parsing rules
+        #
-        util.AnyEndpointMatcher(role="admins", defaultDeny=False),
+        # defaultDeny=False: if user does not have the admin role, we continue
+        # parsing rules
+        util.AnyEndpointMatcher(role='admins', defaultDeny=False),
 
         # Defaulting old config, everyone can stop build.
-        util.StopBuildEndpointMatcher(role="stranger"),
+        util.StopBuildEndpointMatcher(role='stranger'),
-        util.ForceBuildEndpointMatcher(role="stranger"),
+        util.ForceBuildEndpointMatcher(role='stranger'),
-        util.RebuildBuildEndpointMatcher(role="stranger"),
+        util.RebuildBuildEndpointMatcher(role='stranger'),
-        util.EnableSchedulerEndpointMatcher(role="admins"),
+        util.EnableSchedulerEndpointMatcher(role='admins'),
 
-        # if future Buildbot implement new control, we are safe with this last rule
+        # If future Buildbot implement new control, we are safe with this last
+        # rule.
+        #
         # NOTE: This prevents us from cancelling queue, so disabled for now.
-        # util.AnyControlEndpointMatcher(role="admins")
+        # util.AnyControlEndpointMatcher(role='admins')
     ],
     roleMatchers=[
-        RolesFromOwner(role="owner"),
+        RolesFromOwner(role='owner'),
         StrangerRoles(),
     ])
 c['www']['authz'] = authz
-
-# PROJECT IDENTITY
-c['projectName'] = "Blender"
-c['projectURL'] = "https://www.blender.org"
-
-# Buildbot information
-c['buildbotURL'] = "https://builder.blender.org/admin/"
-c['buildbotNetUsageData'] = 'basic'
-
-# Various
-c['db_url'] = "sqlite:///state.sqlite"
-
-c['title'] = "Blender"
-c['titleURL'] = "https://builder.blender.org/"
-
-# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
-c['buildbotNetUsageData'] = None

master_private_template.py

@@ -1,5 +1,18 @@
-slaves = [
+workers = [
-    {'name': 'linux_glibc219_x86_64_chroot',
+    {'name': 'linux_something_something',
-     'password': 'barbarianna',
+     'password': 'something',
-     'builders': ['linux_glibc219_x86_64_cmake']}
+     'platform': 'linux_centos7',
+     'platform_short': 'linux'},
+    {'name': 'macOS_10_15_something_something',
+     'password': 'something',
+     'platform': 'macOS_10_15',
+     'platform_short': 'macos'},
+    {'name': 'macOS_10_13_something_something',
+     'password': 'something',
+     'platform': 'macOS_10_13',
+     'platform_short': 'macos'},
+    {'name': 'windows_something_something',
+     'password': 'something',
+     'platform': 'windows',
+     'platform_short': 'windows'}
 ]

master_unpack.py

@@ -21,129 +21,202 @@
 
 # <pep8 compliant>
 
+import argparse
 import os
 import shutil
 import sys
+import time
 import zipfile
 
+DOWNLOAD_DIR = "/data/www/vhosts/builder.blender.org/webroot/download/"
 
-# extension stripping
+# Parse package filename to extract branch and platform
-def strip_extension(filename):
+class Package:
-    extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
+    def __init__(self, zipname):
+        self.zipname = zipname
+        self.filename = os.path.basename(zipname)
+        self.version = ""
+        self.branch = ""
+        self.platform = ""
+        self.parsed = False
+        self.parse_filename()
 
-    for ext in extensions:
+    # extension stripping
-        if filename.endswith(ext):
+    def _strip_extension(self, filename):
-            filename = filename[:-len(ext)]
+        filename, _ = os.path.splitext(filename)
+        if filename.endswith('.tar'):
+            filename, _ = os.path.splitext(filename)
 
-    return filename
+        return filename
 
+    # extract platform from package name
+    def parse_filename(self):
+        # Name is one one of:
+        # blender-version-platform: official release
+        # blender-version-hash-platform: daily build
+        # branch-blender-version-hash-platform: branch build
+        filename = self._strip_extension(self.filename)
+        tokens = filename.split("-")
 
-# extract platform from package name
+        if len(tokens) < 3:
-def get_platform(filename):
+            return
-    # name is blender-version-platform.extension. we want to get the
+        if "blender" not in tokens:
-    # platform out, but there may be some variations, so we fiddle a
+            return
-    # bit to handle current and hopefully future names
-    filename = strip_extension(filename)
-    filename = strip_extension(filename)
 
-    tokens = filename.split("-")
+        blender_index = tokens.index("blender")
-    platforms = ('osx', 'mac', 'bsd',
+        if not (blender_index + 2 < len(tokens)):
-                 'win', 'linux', 'source',
+            return
-                 'irix', 'solaris', 'mingw')
-    platform_tokens = []
-    found = False
 
-    for i, token in enumerate(tokens):
+        # Detect branch.
-        if not found:
+        branch = "-".join(tokens[:blender_index])
-            for platform in platforms:
-                if platform in token.lower():
-                    found = True
-                    break
-
-        if found:
-            platform_tokens += [token]
-
-    return '-'.join(platform_tokens)
-
-
-def get_branch(filename):
-    tokens = filename.split("-")
-    branch = ""
-
-    for token in tokens:
-        if token == "blender":
-            return branch
 
+        # Detect version.
         if branch == "":
-            branch = token
+            version = tokens[blender_index + 1]
+            if len(tokens) == 3:
+                version += " release"
         else:
-            branch = branch + "-" + token
+            version = ""
 
-    return ""
+        # Detect platform.
+        platform = tokens[-1]
+        platforms = ('osx', 'mac', 'bsd',
+                     'win', 'linux', 'source',
+                     'irix', 'solaris', 'mingw')
+        platform_found = False
+        for name in platforms:
+            if platform.lower().startswith(name):
+                platform_found = True
+        if not platform_found:
+            return
 
-# get filename
+        self.version = version
-if len(sys.argv) < 2:
+        self.platform = platform
-    sys.stderr.write("Not enough arguments, expecting file to unpack\n")
+        self.branch = branch
-    sys.exit(1)
+        self.parsed = True
 
-filename = sys.argv[1]
+def get_download_dir(branch):
+    if not branch or branch == 'master':
+        directory = DOWNLOAD_DIR
+    else:
+        directory = os.path.join(DOWNLOAD_DIR, branch)
+        os.makedirs(directory, exist_ok=True)
+        os.chmod(directory, 0o755)
 
-# open zip file
+    return directory
-if not os.path.exists(filename):
-    sys.stderr.write("File %r not found.\n" % filename)
-    sys.exit(1)
 
-try:
+def open_zipfile(filename):
-    z = zipfile.ZipFile(filename, "r")
+    # Open zip file
-except Exception as ex:
+    if not os.path.exists(filename):
-    sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
+        sys.stderr.write("File %r not found.\n" % filename)
-    sys.exit(1)
+        sys.exit(1)
 
-if len(z.namelist()) != 1:
+    try:
-    sys.stderr.write("Expected one file in %r." % filename)
+        return zipfile.ZipFile(filename, "r")
-    sys.exit(1)
+    except Exception as ex:
+        sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
+        sys.exit(1)
 
-package = z.namelist()[0]
+def get_zipfile_packages(zipfile):
-packagename = os.path.basename(package)
+    # List packages in zip file
+    packages = [Package(zipname) for zipname in zipfile.namelist()]
+    for package in packages:
+        if not package.parsed:
+            sys.stderr.write("Failed to parse package filename: %s\n" % str(package.filename))
+            sys.exit(1)
+    if len(packages) == 0:
+        sys.stderr.write('Empty zip file\n')
+        sys.exit(1)
+    return packages
 
-# detect platform and branch
+def get_branch_platform(packages):
-platform = get_platform(packagename)
+    # Extract branch and platform names
-branch = get_branch(packagename)
+    branch = packages[0].branch
+    platform = packages[0].platform
+    version = packages[0].version
 
-if platform == '':
+    if platform == '':
-    sys.stderr.write('Failed to detect platform ' +
+        sys.stderr.write('Failed to detect platform ' +
-        'from package: %r\n' % packagename)
+            'from package: %r\n' % packages[0].filename)
-    sys.exit(1)
+        sys.exit(1)
 
-# extract
+    for package in packages:
-download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
+        if package.branch != branch or package.platform != platform or package.version != version:
-if not branch or branch == 'master':
+            sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
-    directory = download_prefix
+            sys.exit(1)
-elif branch == 'experimental-build':
-    directory = os.path.join(download_prefix, "experimental")
-else:
-    directory = download_prefix
 
-try:
+    return branch, platform, version
-    filename = os.path.join(directory, packagename)
-    zf = z.open(package)
-    f = open(filename, "wb")
 
-    shutil.copyfileobj(zf, f)
+def extract_zipfile_packages(zipfile, packages, branch):
-    os.chmod(filename, 0o644)
+    # Extract packages from zip file
+    directory = get_download_dir(branch)
 
-    zf.close()
+    for package in packages:
-    z.close()
+        filepath = os.path.join(directory, package.filename)
-except Exception as ex:
-    sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
-    sys.exit(1)
 
-# remove other files from the same platform and branch
+        try:
-try:
+            zf = zipfile.open(package.zipname)
-    for f in os.listdir(directory):
+            f = open(filepath, "wb")
-        if get_platform(f) == platform and get_branch(f) == branch:
+
-            if f != packagename:
+            shutil.copyfileobj(zf, f)
-                os.remove(os.path.join(directory, f))
+            os.chmod(filepath, 0o644)
-except Exception as ex:
+
-    sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
+            zf.close()
-    sys.exit(1)
+        except Exception as ex:
+            sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
+            sys.exit(1)
+
+def remove_replaced_packages(branch, platform, version, new_packages):
+    # Remove other files from the same platform and branch that are replaced
+    # by the new packages.
+    directory = get_download_dir(branch)
+
+    for filename in os.listdir(directory):
+        package = Package(filename)
+        if package.platform == platform and package.branch == branch and package.version == version:
+            is_new_package = False
+            for new_package in new_packages:
+                if package.filename == new_package.filename:
+                    is_new_package = True
+
+            if not is_new_package:
+                try:
+                    print("Removing older package version", filename)
+                    os.remove(os.path.join(directory, filename))
+                except Exception as ex:
+                    sys.stderr.write('Failed to remove replaced package: %s\n' % str(ex))
+
+def remove_old_packages():
+    # Remove any branch packages that are 100 days or older.
+    cutoff_time = time.time() - 100 * 86400
+
+    for dirname in os.listdir(DOWNLOAD_DIR):
+        dirpath = os.path.join(DOWNLOAD_DIR, dirname)
+        if not os.path.isdir(dirpath):
+            continue
+
+        for filename in os.listdir(dirpath):
+            filepath = os.path.join(dirpath, filename)
+            if not os.path.isfile(filepath):
+                continue
+
+            file_mtime = os.stat(filepath).st_mtime
+            if file_mtime < cutoff_time:
+                try:
+                    print("Removing old branch build", filename)
+                    os.remove(filepath)
+                except Exception as ex:
+                    sys.stderr.write('Failed to remove old package: %s\n' % str(ex))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filename')
+    args = parser.parse_args()
+
+    with open_zipfile(args.filename) as zipfile:
+        packages = get_zipfile_packages(zipfile)
+        branch, platform, version = get_branch_platform(packages)
+        extract_zipfile_packages(zipfile, packages, branch)
+        remove_replaced_packages(branch, platform, version, packages)
+
+    remove_old_packages()