Additional fixes

Ref D5545

master.cfg

@@ -6,15 +6,6 @@
 # List of the branches being built automatically overnight
 NIGHT_SCHEDULE_BRANCHES = ["master"]
 
-# List of the branches available for force build
-FORCE_SCHEDULE_BRANCHES = ["master", "experimental-build"]
-
-# List of branches availble for addons branch selection.
-ADDONS_BRANCHES = ["master"]
-
-# List of branches availble for libraries branch selection.
-LIB_BRANCHES = ["trunk"]
-
 # Dictionary that the buildmaster pays attention to.
 c = BuildmasterConfig = {}
 
@@ -50,10 +41,6 @@ c['change_source'] = GitPoller('git://git.blender.org/blender.git',
 
 all_repositories = {
     r'git://git.blender.org/blender.git': 'blender',
-    r'git://git.blender.org/blender-translations.git': 'blender-translations',
-    r'git://git.blender.org/blender-addons.git': 'blender-addons',
-    r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
-    r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
 }
 
@@ -85,11 +72,17 @@ def schedule_force_build(name):
         builderNames=[name],
         codebases=[forcesched.CodebaseParameter(
                 codebase="blender",
-                branch=forcesched.ChoiceStringParameter(
+                branch=forcesched.StringParameter(
                     name="branch",
-                    choices=FORCE_SCHEDULE_BRANCHES,
+                    label="Branch:",
                     default="master"),
-                # Do not hide revision, can be handy!
+                # Hide revision. We don't want to allow anyone to overwrite the
+                # master build with an older version. Could be added back once we
+                # have authentication.
+                revision=forcesched.FixedParameter(
+                    name="revision",
+                    default="",
+                    hide=True),
                 repository=forcesched.FixedParameter(
                     name="repository",
                     default="",
@@ -98,61 +91,6 @@ def schedule_force_build(name):
                     name="project",
                     default="",
                     hide=True)),
-                   # For now, hide other codebases.
-                   forcesched.CodebaseParameter(
-                       hide=True,
-                       codebase="blender-translations"),
-                   forcesched.CodebaseParameter(
-                       codebase="blender-addons",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=ADDONS_BRANCHES,
-                           default="master"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-                   forcesched.CodebaseParameter(
-                       codebase="blender-addons-contrib",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=ADDONS_BRANCHES,
-                           default="master"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
-                   forcesched.CodebaseParameter(
-                       hide=True,
-                       codebase="blender-dev-tools"),
-                   forcesched.CodebaseParameter(
-                       codebase="lib svn",
-                       branch=forcesched.ChoiceStringParameter(
-                           name="branch",
-                           choices=LIB_BRANCHES,
-                           default="trunk"),
-                       repository=forcesched.FixedParameter(name="repository",
-                                                            default="",
-                                                            hide=True),
-                       project=forcesched.FixedParameter(name="project",
-                                                         default="",
-                                                         hide=True),
-                       revision=forcesched.FixedParameter(name="revision",
-                                                          default="",
-                                                          hide=True),
-                   ),
         ],
         properties=[]))
 
@@ -165,22 +103,11 @@ def schedule_nightly_build(name, hour, minute=0):
         scheduler_name = "nightly_" + name
         if current_branch:
             scheduler_name += ' ' + current_branch
-        addons_branch = "master"
         c['schedulers'].append(timed.Nightly(
             name=scheduler_name,
             codebases={
                 "blender": {"repository": "",
-                            "branch": current_branch},
-                "blender-translations": {"repository": "",
-                                         "branch": "master"},
-                "blender-addons": {"repository": "",
-                                   "branch": addons_branch},
-                "blender-addons-contrib": {"repository": "",
-                                           "branch": addons_branch},
-                "blender-dev-tools": {"repository": "",
-                                      "branch": "master"},
-                "lib svn": {"repository": "",
-                            "branch": "trunk"}},
+                            "branch": current_branch}},
             branch=current_branch,
             builderNames=[name],
             hour=hour,
@@ -209,7 +136,7 @@ c['builders'] = []
 buildernames = []
 
 
-def add_builder(c, name, libdir, factory, branch='',
+def add_builder(c, name, factory, branch='',
                 rsync=False, hour=3, minute=0):
     workernames = []
 
@@ -218,7 +145,7 @@ def add_builder(c, name, libdir, factory, branch='',
             workernames.append(slave['name'])
 
     if workernames:
-        f = factory(name, libdir, branch, rsync)
+        f = factory(name, branch, rsync)
         c['builders'].append(BuilderConfig(name=name,
                                            workernames=workernames,
                                            factory=f,
@@ -231,14 +158,6 @@ def add_builder(c, name, libdir, factory, branch='',
 
 # common steps
 
-def git_submodule_step(submodule):
-    return steps.Git(name=submodule + '.git',
-                     repourl='git://git.blender.org/' + submodule + '.git',
-                     mode='incremental',
-                     codebase=submodule,
-                     workdir=submodule + '.git')
-
-
 def git_step(branch=''):
     if branch:
         return steps.Git(name='blender.git',
@@ -257,29 +176,9 @@ def git_step(branch=''):
                          submodules=True)
 
 
-def git_submodules_update():
-    command = ['git', 'submodule', 'update', '--remote']
-    return ShellCommand(name='Submodules Update',
-                        command=command,
-                        description='updating',
-                        descriptionDone='up to date',
-                        workdir='blender.git')
-
-
-def lib_svn_step(dir):
-    # TODO(sergey): For some reason interpolation is always giving empty branch.
-    # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
-    lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
-    return steps.SVN(name='lib svn ' + dir,
-                     repourl=util.Interpolate(lib_repo + dir),
-                     codebase='lib svn',
-                     mode='incremental',
-                     workdir='lib/' + dir)
-
-
-def rsync_step(id, branch, rsync_script):
+def rsync_step(python_command, id, branch, rsync_script):
     return ShellCommand(name='rsync',
-                        command=['python3', rsync_script, id, branch],
+                        command=[python_command, rsync_script, id, branch],
                         description='uploading',
                         descriptionDone='uploaded',
                         workdir='install')
@@ -287,8 +186,9 @@ def rsync_step(id, branch, rsync_script):
 
 # generic builder
 
-def generic_builder(id, libdir='', branch='', rsync=False):
+def generic_builder(id, branch='', rsync=False):
     filename = 'uploaded/buildbot_upload_' + id + '.zip'
+    update_script = '../blender.git/build_files/buildbot/slave_update.py'
     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
     test_script = '../blender.git/build_files/buildbot/slave_test.py'
     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
@@ -296,32 +196,34 @@ def generic_builder(id, libdir='', branch='', rsync=False):
     unpack_script = 'master_unpack.py'
 
     f = BuildFactory()
-    if libdir != '':
-        f.addStep(lib_svn_step(libdir))
-
-    f.addStep(lib_svn_step('tests'))
-
-    for submodule in ('blender-translations',
-                      'blender-addons',
-                      'blender-addons-contrib',
-                      'blender-dev-tools'):
-        f.addStep(git_submodule_step(submodule))
+    if id.startswith('win'):
+        python_command = 'python'
+    else:
+        python_command = 'python3'
 
     f.addStep(git_step(branch))
-    f.addStep(git_submodules_update())
 
-    f.addStep(Compile(command=['python3', compile_script, id], timeout=3600))
-    f.addStep(Test(command=['python3', test_script, id]))
+    git_branch = branch or Interpolate('%(src:blender:branch)s')
+
+    f.addStep(ShellCommand(
+        name='submodules and libraries update',
+        command=[python_command, update_script, id, git_branch],
+        description='updating',
+        descriptionDone='updated'))
+
+    f.addStep(Compile(
+        command=[python_command, compile_script, id, git_branch],
+        timeout=3600))
+    f.addStep(Test(
+        command=[python_command, test_script, id, git_branch]))
     f.addStep(ShellCommand(
         name='package',
-        command=['python3',
-                 pack_script,
-                 id,
-                 branch or Interpolate('%(src:blender:branch)s')],
+        command=[python_command, pack_script, id, git_branch],
         description='packaging',
         descriptionDone='packaged'))
+
     if rsync:
-        f.addStep(rsync_step(id, branch, rsync_script))
+        f.addStep(rsync_step(python_command, id, git_branch, rsync_script))
     else:
         f.addStep(FileUpload(name='upload',
                              workersrc='buildbot_upload.zip',
@@ -339,11 +241,11 @@ def generic_builder(id, libdir='', branch='', rsync=False):
 
 # Builders
 
-add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
-add_builder(c, 'linux_glibc217_x86_64_cmake', '', generic_builder, hour=1)
+add_builder(c, 'mac_x86_64_10_9_cmake', generic_builder, hour=1)
+add_builder(c, 'linux_glibc217_x86_64_cmake', generic_builder, hour=1)
 # NOTE: Visual Studio 2017 (vc15) is using libraries folder from
 # Visual Studio 2015 (vc14)
-add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=1)
+add_builder(c, 'win64_cmake_vs2017', generic_builder, hour=1)
 
 # HORIZONS
 from datetime import timedelta

master_unpack.py

@@ -21,129 +21,169 @@
 
 # <pep8 compliant>
 
+import argparse
 import os
 import shutil
 import sys
 import zipfile
 
+# Parse package filename to extract branch and platform
+class Package:
+    def __init__(self, zipname):
+        self.zipname = zipname
+        self.filename = os.path.basename(zipname)
+        self.platform = self._get_platform(self.filename)
+        self.branch = self._get_branch(self.filename)
 
-# extension stripping
-def strip_extension(filename):
-    extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
+    # extension stripping
+    def _strip_extension(self, filename):
+        extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
 
-    for ext in extensions:
-        if filename.endswith(ext):
-            filename = filename[:-len(ext)]
+        for ext in extensions:
+            if filename.endswith(ext):
+                filename = filename[:-len(ext)]
 
-    return filename
+        return filename
+
+    # extract platform from package name
+    def _get_platform(self, filename):
+        # name is blender-version-platform.extension. we want to get the
+        # platform out, but there may be some variations, so we fiddle a
+        # bit to handle current and hopefully future names
+        filename = self._strip_extension(filename)
+        filename = self._strip_extension(filename)
+
+        tokens = filename.split("-")
+        platforms = ('osx', 'mac', 'bsd',
+                     'win', 'linux', 'source',
+                     'irix', 'solaris', 'mingw')
+        platform_tokens = []
+        found = False
+
+        for i, token in enumerate(tokens):
+            if not found:
+                for platform in platforms:
+                    if platform in token.lower():
+                        found = True
+                        break
+
+            if found:
+                platform_tokens += [token]
+
+        return '-'.join(platform_tokens)
 
 
-# extract platform from package name
-def get_platform(filename):
-    # name is blender-version-platform.extension. we want to get the
-    # platform out, but there may be some variations, so we fiddle a
-    # bit to handle current and hopefully future names
-    filename = strip_extension(filename)
-    filename = strip_extension(filename)
+    def _get_branch(self, filename):
+        tokens = filename.split("-")
+        branch = ""
 
-    tokens = filename.split("-")
-    platforms = ('osx', 'mac', 'bsd',
-                 'win', 'linux', 'source',
-                 'irix', 'solaris', 'mingw')
-    platform_tokens = []
-    found = False
+        for token in tokens:
+            if token == "blender":
+                return branch
 
-    for i, token in enumerate(tokens):
-        if not found:
-            for platform in platforms:
-                if platform in token.lower():
-                    found = True
-                    break
+            if branch == "":
+                branch = token
+            else:
+                branch = branch + "-" + token
 
-        if found:
-            platform_tokens += [token]
-
-    return '-'.join(platform_tokens)
+        return ""
 
 
-def get_branch(filename):
-    tokens = filename.split("-")
-    branch = ""
+def get_download_dir(branch):
+    download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
+    if not branch or branch == 'master':
+        directory = download_prefix
+    elif branch == 'experimental-build':
+        directory = os.path.join(download_prefix, "experimental")
+    else:
+        directory = os.path.join(download_prefix, branch)
+        os.makedirs(directory, exist_ok=True)
 
-    for token in tokens:
-        if token == "blender":
-            return branch
+    return directory
 
-        if branch == "":
-            branch = token
-        else:
-            branch = branch + "-" + token
+def open_zipfile(filename):
+    # Open zip file
+    if not os.path.exists(filename):
+        sys.stderr.write("File %r not found.\n" % filename)
+        sys.exit(1)
 
-    return ""
+    try:
+        return zipfile.ZipFile(filename, "r")
+    except Exception as ex:
+        sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
+        sys.exit(1)
 
-# get filename
-if len(sys.argv) < 2:
-    sys.stderr.write("Not enough arguments, expecting file to unpack\n")
-    sys.exit(1)
+def get_zipfile_packages(zipfile):
+    # List packages in zip file
+    packages = [Package(zipname) for zipname in zipfile.namelist()]
+    if len(packages) == 0:
+        sys.stderr.write('Empty zip file\n')
+        sys.exit(1)
+    return packages
 
-filename = sys.argv[1]
+def get_branch_platform(packages):
+    # Extract branch and platform names
+    branch = packages[0].branch
+    platform = packages[0].platform
 
-# open zip file
-if not os.path.exists(filename):
-    sys.stderr.write("File %r not found.\n" % filename)
-    sys.exit(1)
+    if platform == '':
+        sys.stderr.write('Failed to detect platform ' +
+            'from package: %r\n' % packages[0].filename)
+        sys.exit(1)
 
-try:
-    z = zipfile.ZipFile(filename, "r")
-except Exception as ex:
-    sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
-    sys.exit(1)
+    for package in packages:
+        if package.branch != branch or package.platform != platform:
+            sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
+            sys.exit(1)
 
-if len(z.namelist()) != 1:
-    sys.stderr.write("Expected one file in %r." % filename)
-    sys.exit(1)
+    return branch, platform
 
-package = z.namelist()[0]
-packagename = os.path.basename(package)
+def extract_zipfile_packages(zipfile, packages, branch):
+    # Extract packages from zip file
+    directory = get_download_dir(branch)
 
-# detect platform and branch
-platform = get_platform(packagename)
-branch = get_branch(packagename)
+    for package in packages:
+        filepath = os.path.join(directory, package.filename)
 
-if platform == '':
-    sys.stderr.write('Failed to detect platform ' +
-        'from package: %r\n' % packagename)
-    sys.exit(1)
+        try:
+            zf = zipfile.open(package.zipname)
+            f = open(filepath, "wb")
 
-# extract
-download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
-if not branch or branch == 'master':
-    directory = download_prefix
-elif branch == 'experimental-build':
-    directory = os.path.join(download_prefix, "experimental")
-else:
-    directory = download_prefix
+            shutil.copyfileobj(zf, f)
+            os.chmod(filepath, 0o644)
 
-try:
-    filename = os.path.join(directory, packagename)
-    zf = z.open(package)
-    f = open(filename, "wb")
+            zf.close()
+        except Exception as ex:
+            sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
+            sys.exit(1)
 
-    shutil.copyfileobj(zf, f)
-    os.chmod(filename, 0o644)
+def remove_older_packages(branch, platform, new_packages):
+    # Remove other files from the same platform and branch
+    directory = get_download_dir(branch)
 
-    zf.close()
-    z.close()
-except Exception as ex:
-    sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
-    sys.exit(1)
+    for filename in os.listdir(directory):
+        package = Package(filename)
+        if package.platform == platform and package.branch == branch:
+            is_new_package = False
+            for new_package in new_packages:
+                if package.filename == new_package.filename:
+                    is_new_package = True
 
-# remove other files from the same platform and branch
-try:
-    for f in os.listdir(directory):
-        if get_platform(f) == platform and get_branch(f) == branch:
-            if f != packagename:
-                os.remove(os.path.join(directory, f))
-except Exception as ex:
-    sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
-    sys.exit(1)
+            if not is_new_package:
+                try:
+                    os.remove(os.path.join(directory, filename))
+                except Exception as ex:
+                    sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filename')
+    args = parser.parse_args()
+
+    with open_zipfile(args.filename) as zipfile:
+        packages = get_zipfile_packages(zipfile)
+        branch, platform = get_branch_platform(packages)
+        extract_zipfile_packages(zipfile, packages, branch)
+
+    remove_older_packages(branch, platform, packages)