Additional fixes

Ref D5545

master.cfg

@@ -6,15 +6,6 @@
 # List of the branches being built automatically overnight
 NIGHT_SCHEDULE_BRANCHES = ["master"]
 
-# List of the branches available for force build
-FORCE_SCHEDULE_BRANCHES = ["master", "experimental-build"]
-
-# List of branches availble for addons branch selection.
-ADDONS_BRANCHES = ["master"]
-
-# List of branches availble for libraries branch selection.
-LIB_BRANCHES = ["trunk"]
-
 # Dictionary that the buildmaster pays attention to.
 c = BuildmasterConfig = {}
 
@@ -50,10 +41,6 @@ c['change_source'] = GitPoller('git://git.blender.org/blender.git',
 
 all_repositories = {
     r'git://git.blender.org/blender.git': 'blender',
-    r'git://git.blender.org/blender-translations.git': 'blender-translations',
-    r'git://git.blender.org/blender-addons.git': 'blender-addons',
-    r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
-    r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
 }
 
@@ -85,11 +72,17 @@ def schedule_force_build(name):
         builderNames=[name],
         codebases=[forcesched.CodebaseParameter(
                 codebase="blender",
-                branch=forcesched.ChoiceStringParameter(
+                branch=forcesched.StringParameter(
                     name="branch",
-                    choices=FORCE_SCHEDULE_BRANCHES,
+                    label="Branch:",
                     default="master"),
-                # Do not hide revision, can be handy!
+                # Hide revision. We don't want to allow anyone to overwrite the
+                # master build with an older version. Could be added back once we
+                # have authentication.
+                revision=forcesched.FixedParameter(
+                    name="revision",
+                    default="",
+                    hide=True),
                 repository=forcesched.FixedParameter(
                     name="repository",
                     default="",
@@ -98,61 +91,6 @@ def schedule_force_build(name):
                     name="project",
                     default="",
                     hide=True)),
-                   # For now, hide other codebases.
-                   forcesched.CodebaseParameter(
-                       hide=True,
-                       codebase="blender-translations"),
-                   forcesched.CodebaseParameter(
-                       codebase="blender-addons",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=ADDONS_BRANCHES,
-                           default="master"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-                   forcesched.CodebaseParameter(
-                       codebase="blender-addons-contrib",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=ADDONS_BRANCHES,
-                           default="master"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-                   forcesched.CodebaseParameter(
-                       hide=True,
-                       codebase="blender-dev-tools"),
-                   forcesched.CodebaseParameter(
-                       codebase="lib svn",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=LIB_BRANCHES,
-                           default="trunk"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
         ],
         properties=[]))
 
@@ -165,22 +103,11 @@ def schedule_nightly_build(name, hour, minute=0):
         scheduler_name = "nightly_" + name
         if current_branch:
             scheduler_name += ' ' + current_branch
-        addons_branch = "master"
         c['schedulers'].append(timed.Nightly(
             name=scheduler_name,
             codebases={
                 "blender": {"repository": "",
-                            "branch": current_branch},
+                            "branch": current_branch}},
-                "blender-translations": {"repository": "",
-                                         "branch": "master"},
-                "blender-addons": {"repository": "",
-                                   "branch": addons_branch},
-                "blender-addons-contrib": {"repository": "",
-                                           "branch": addons_branch},
-                "blender-dev-tools": {"repository": "",
-                                      "branch": "master"},
-                "lib svn": {"repository": "",
-                            "branch": "trunk"}},
             branch=current_branch,
             builderNames=[name],
             hour=hour,
@@ -209,7 +136,7 @@ c['builders'] = []
 buildernames = []
 
 
-def add_builder(c, name, libdir, factory, branch='',
+def add_builder(c, name, factory, branch='',
                 rsync=False, hour=3, minute=0):
     workernames = []
 
@@ -218,7 +145,7 @@ def add_builder(c, name, libdir, factory, branch='',
             workernames.append(slave['name'])
 
     if workernames:
-        f = factory(name, libdir, branch, rsync)
+        f = factory(name, branch, rsync)
         c['builders'].append(BuilderConfig(name=name,
                                            workernames=workernames,
                                            factory=f,
@@ -231,14 +158,6 @@ def add_builder(c, name, libdir, factory, branch='',
 
 # common steps
 
-def git_submodule_step(submodule):
-    return steps.Git(name=submodule + '.git',
-                     repourl='git://git.blender.org/' + submodule + '.git',
-                     mode='incremental',
-                     codebase=submodule,
-                     workdir=submodule + '.git')
-
-
 def git_step(branch=''):
     if branch:
         return steps.Git(name='blender.git',
@@ -257,29 +176,9 @@ def git_step(branch=''):
                          submodules=True)
 
 
-def git_submodules_update():
+def rsync_step(python_command, id, branch, rsync_script):
-    command = ['git', 'submodule', 'update', '--remote']
-    return ShellCommand(name='Submodules Update',
-                        command=command,
-                        description='updating',
-                        descriptionDone='up to date',
-                        workdir='blender.git')
-
-
-def lib_svn_step(dir):
-    # TODO(sergey): For some reason interpolation is always giving empty branch.
-    # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
-    lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
-    return steps.SVN(name='lib svn ' + dir,
-                     repourl=util.Interpolate(lib_repo + dir),
-                     codebase='lib svn',
-                     mode='incremental',
-                     workdir='lib/' + dir)
-
-
-def rsync_step(id, branch, rsync_script):
     return ShellCommand(name='rsync',
-                        command=['python3', rsync_script, id, branch],
+                        command=[python_command, rsync_script, id, branch],
                         description='uploading',
                         descriptionDone='uploaded',
                         workdir='install')
@@ -287,8 +186,9 @@ def rsync_step(id, branch, rsync_script):
 
 # generic builder
 
-def generic_builder(id, libdir='', branch='', rsync=False):
+def generic_builder(id, branch='', rsync=False):
     filename = 'uploaded/buildbot_upload_' + id + '.zip'
+    update_script = '../blender.git/build_files/buildbot/slave_update.py'
     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
     test_script = '../blender.git/build_files/buildbot/slave_test.py'
     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
@@ -296,32 +196,34 @@ def generic_builder(id, libdir='', branch='', rsync=False):
     unpack_script = 'master_unpack.py'
 
     f = BuildFactory()
-    if libdir != '':
+    if id.startswith('win'):
-        f.addStep(lib_svn_step(libdir))
+        python_command = 'python'
-
+    else:
-    f.addStep(lib_svn_step('tests'))
+        python_command = 'python3'
-
-    for submodule in ('blender-translations',
-                      'blender-addons',
-                      'blender-addons-contrib',
-                      'blender-dev-tools'):
-        f.addStep(git_submodule_step(submodule))
 
     f.addStep(git_step(branch))
-    f.addStep(git_submodules_update())
 
-    f.addStep(Compile(command=['python3', compile_script, id], timeout=3600))
+    git_branch = branch or Interpolate('%(src:blender:branch)s')
-    f.addStep(Test(command=['python3', test_script, id]))
+
+    f.addStep(ShellCommand(
+        name='submodules and libraries update',
+        command=[python_command, update_script, id, git_branch],
+        description='updating',
+        descriptionDone='updated'))
+
+    f.addStep(Compile(
+        command=[python_command, compile_script, id, git_branch],
+        timeout=3600))
+    f.addStep(Test(
+        command=[python_command, test_script, id, git_branch]))
     f.addStep(ShellCommand(
         name='package',
-        command=['python3',
+        command=[python_command, pack_script, id, git_branch],
-                 pack_script,
-                 id,
-                 branch or Interpolate('%(src:blender:branch)s')],
         description='packaging',
         descriptionDone='packaged'))
+
     if rsync:
-        f.addStep(rsync_step(id, branch, rsync_script))
+        f.addStep(rsync_step(python_command, id, git_branch, rsync_script))
     else:
         f.addStep(FileUpload(name='upload',
                              workersrc='buildbot_upload.zip',
@@ -339,11 +241,11 @@ def generic_builder(id, libdir='', branch='', rsync=False):
 
 # Builders
 
-add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
+add_builder(c, 'mac_x86_64_10_9_cmake', generic_builder, hour=1)
-add_builder(c, 'linux_glibc217_x86_64_cmake', '', generic_builder, hour=1)
+add_builder(c, 'linux_glibc217_x86_64_cmake', generic_builder, hour=1)
 # NOTE: Visual Studio 2017 (vc15) is using libraries folder from
 # Visual Studio 2015 (vc14)
-add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=1)
+add_builder(c, 'win64_cmake_vs2017', generic_builder, hour=1)
 
 # HORIZONS
 from datetime import timedelta

master_unpack.py

@@ -21,14 +21,22 @@
 
 # <pep8 compliant>
 
+import argparse
 import os
 import shutil
 import sys
 import zipfile
 
+# Parse package filename to extract branch and platform
+class Package:
+    def __init__(self, zipname):
+        self.zipname = zipname
+        self.filename = os.path.basename(zipname)
+        self.platform = self._get_platform(self.filename)
+        self.branch = self._get_branch(self.filename)
 
     # extension stripping
-def strip_extension(filename):
+    def _strip_extension(self, filename):
         extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
 
         for ext in extensions:
@@ -37,14 +45,13 @@ def strip_extension(filename):
 
         return filename
 
-
     # extract platform from package name
-def get_platform(filename):
+    def _get_platform(self, filename):
         # name is blender-version-platform.extension. we want to get the
         # platform out, but there may be some variations, so we fiddle a
         # bit to handle current and hopefully future names
-    filename = strip_extension(filename)
+        filename = self._strip_extension(filename)
-    filename = strip_extension(filename)
+        filename = self._strip_extension(filename)
 
         tokens = filename.split("-")
         platforms = ('osx', 'mac', 'bsd',
@@ -66,7 +73,7 @@ def get_platform(filename):
         return '-'.join(platform_tokens)
 
 
-def get_branch(filename):
+    def _get_branch(self, filename):
         tokens = filename.split("-")
         branch = ""
 
@@ -81,69 +88,102 @@ def get_branch(filename):
 
         return ""
 
-# get filename
-if len(sys.argv) < 2:
-    sys.stderr.write("Not enough arguments, expecting file to unpack\n")
-    sys.exit(1)
 
-filename = sys.argv[1]
+def get_download_dir(branch):
-
-# open zip file
-if not os.path.exists(filename):
-    sys.stderr.write("File %r not found.\n" % filename)
-    sys.exit(1)
-
-try:
-    z = zipfile.ZipFile(filename, "r")
-except Exception as ex:
-    sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
-    sys.exit(1)
-
-if len(z.namelist()) != 1:
-    sys.stderr.write("Expected one file in %r." % filename)
-    sys.exit(1)
-
-package = z.namelist()[0]
-packagename = os.path.basename(package)
-
-# detect platform and branch
-platform = get_platform(packagename)
-branch = get_branch(packagename)
-
-if platform == '':
-    sys.stderr.write('Failed to detect platform ' +
-        'from package: %r\n' % packagename)
-    sys.exit(1)
-
-# extract
     download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
     if not branch or branch == 'master':
         directory = download_prefix
     elif branch == 'experimental-build':
         directory = os.path.join(download_prefix, "experimental")
     else:
-    directory = download_prefix
+        directory = os.path.join(download_prefix, branch)
+        os.makedirs(directory, exist_ok=True)
+
+    return directory
+
+def open_zipfile(filename):
+    # Open zip file
+    if not os.path.exists(filename):
+        sys.stderr.write("File %r not found.\n" % filename)
+        sys.exit(1)
 
     try:
-    filename = os.path.join(directory, packagename)
+        return zipfile.ZipFile(filename, "r")
-    zf = z.open(package)
+    except Exception as ex:
-    f = open(filename, "wb")
+        sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
+        sys.exit(1)
+
+def get_zipfile_packages(zipfile):
+    # List packages in zip file
+    packages = [Package(zipname) for zipname in zipfile.namelist()]
+    if len(packages) == 0:
+        sys.stderr.write('Empty zip file\n')
+        sys.exit(1)
+    return packages
+
+def get_branch_platform(packages):
+    # Extract branch and platform names
+    branch = packages[0].branch
+    platform = packages[0].platform
+
+    if platform == '':
+        sys.stderr.write('Failed to detect platform ' +
+            'from package: %r\n' % packages[0].filename)
+        sys.exit(1)
+
+    for package in packages:
+        if package.branch != branch or package.platform != platform:
+            sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
+            sys.exit(1)
+
+    return branch, platform
+
+def extract_zipfile_packages(zipfile, packages, branch):
+    # Extract packages from zip file
+    directory = get_download_dir(branch)
+
+    for package in packages:
+        filepath = os.path.join(directory, package.filename)
+
+        try:
+            zf = zipfile.open(package.zipname)
+            f = open(filepath, "wb")
 
             shutil.copyfileobj(zf, f)
-    os.chmod(filename, 0o644)
+            os.chmod(filepath, 0o644)
 
             zf.close()
-    z.close()
         except Exception as ex:
             sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
             sys.exit(1)
 
-# remove other files from the same platform and branch
+def remove_older_packages(branch, platform, new_packages):
+    # Remove other files from the same platform and branch
+    directory = get_download_dir(branch)
+
+    for filename in os.listdir(directory):
+        package = Package(filename)
+        if package.platform == platform and package.branch == branch:
+            is_new_package = False
+            for new_package in new_packages:
+                if package.filename == new_package.filename:
+                    is_new_package = True
+
+            if not is_new_package:
                 try:
-    for f in os.listdir(directory):
+                    os.remove(os.path.join(directory, filename))
-        if get_platform(f) == platform and get_branch(f) == branch:
-            if f != packagename:
-                os.remove(os.path.join(directory, f))
                 except Exception as ex:
                     sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
-    sys.exit(1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filename')
+    args = parser.parse_args()
+
+    with open_zipfile(args.filename) as zipfile:
+        packages = get_zipfile_packages(zipfile)
+        branch, platform = get_branch_platform(packages)
+        extract_zipfile_packages(zipfile, packages, branch)
+
+    remove_older_packages(branch, platform, packages)