Buildbot: refactor all build slave code

* Move common code into buildbot_utils.py
* Remove legacy code from removed builders
* Split code into smaller functions

Differential Revision: https://developer.blender.org/D5642

build_files/buildbot/buildbot_utils.py

@@ -0,0 +1,92 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+# <pep8 compliant>
+
+import argparse
+import os
+import subprocess
+import sys
+
+class Builder:
+    def __init__(self, name, branch):
+        self.name = name
+        self.branch = branch
+
+        # Buildbot runs from build/ directory
+        self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git'))
+        self.build_dir = os.path.abspath(os.path.join('..', 'build', name))
+        self.install_dir = os.path.abspath(os.path.join('..', 'install', name))
+        self.upload_dir = os.path.abspath(os.path.join('..', 'install'))
+
+        # Detect platform
+        if name.startswith('mac'):
+            self.platform = 'mac'
+            self.command_prefix =  []
+        elif name.startswith('linux'):
+            self.platform = 'linux'
+            self.command_prefix =  ['scl', 'enable', 'devtoolset-6', '--']
+        elif name.startswith('win'):
+            self.platform = 'win'
+            self.command_prefix =  []
+        else:
+            raise ValueError('Unkonw platform for builder ' + self.platform)
+
+        # Always 64 bit now
+        self.bits = 64
+
+def create_builder_from_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('builder_name')
+    parser.add_argument('branch', default='master', nargs='?')
+    args = parser.parse_args()
+    return Builder(args.builder_name, args.branch)
+
+
+class VersionInfo:
+    def __init__(self, builder):
+        # Get version information
+        buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h")
+        blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
+
+        version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION'))
+        self.version = "%d.%d" % (version_number // 100, version_number % 100)
+        self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
+
+    def _parse_header_file(self, filename, define):
+        import re
+        regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
+        with open(filename, "r") as file:
+            for l in file:
+                match = regex.match(l)
+                if match:
+                    return match.group(1)
+        return None
+
+
+def call(cmd, env=None, exit_on_error=True):
+    print(' '.join(cmd))
+
+    # Flush to ensure correct order output on Windows.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    retcode = subprocess.call(cmd, env=env)
+    if exit_on_error and retcode != 0:
+        sys.exit(retcode)
+    return retcode

build_files/buildbot/slave_compile.py

@@ -18,149 +18,80 @@
 
 # <pep8 compliant>
 
+import buildbot_utils
 import os
-import subprocess
-import sys
 import shutil
 
-# get builder name
+def get_cmake_options(builder):
-if len(sys.argv) < 2:
+    config_file = "build_files/cmake/config/blender_release.cmake"
-    sys.stderr.write("Not enough arguments, expecting builder name\n")
+    options = ['-DCMAKE_BUILD_TYPE:STRING=Release', '-DWITH_GTESTS=ON']
-    sys.exit(1)
 
-builder = sys.argv[1]
+    if builder.platform == 'mac':
+        options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
+        options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
+    elif builder.platform == 'win':
+        options.extend(['-G', 'Visual Studio 15 2017 Win64'])
+    elif builder.platform == 'linux':
+        config_file = "build_files/buildbot/config/blender_linux.cmake"
 
-# we run from build/ directory
+    options.append("-C" + os.path.join(builder.blender_dir, config_file))
-blender_dir = os.path.join('..', 'blender.git')
+    options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir))
 
+    return options
 
-def parse_header_file(filename, define):
+def update_git(builder):
-    import re
-    regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
-    with open(filename, "r") as file:
-        for l in file:
-            match = regex.match(l)
-            if match:
-                return match.group(1)
-    return None
-
-if 'cmake' in builder:
-    # cmake
-
-    # Some fine-tuning configuration
-    blender_dir = os.path.abspath(blender_dir)
-    build_dir = os.path.abspath(os.path.join('..', 'build', builder))
-    install_dir = os.path.abspath(os.path.join('..', 'install', builder))
-    targets = ['blender']
-    command_prefix = []
-
-    bits = 64
-
-    # Config file to be used (relative to blender's sources root)
-    cmake_config_file = "build_files/cmake/config/blender_release.cmake"
-
-    # Set build options.
-    cmake_options = []
-    cmake_extra_options = ['-DCMAKE_BUILD_TYPE:STRING=Release',
-                           '-DWITH_GTESTS=ON']
-
-    if builder.startswith('mac'):
-        # Set up OSX architecture
-        if builder.endswith('x86_64_10_9_cmake'):
-            cmake_extra_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
-        cmake_extra_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
-
-    elif builder.startswith('win'):
-        if builder.startswith('win64'):
-            cmake_options.extend(['-G', 'Visual Studio 15 2017 Win64'])
-        elif builder.startswith('win32'):
-            bits = 32
-            cmake_options.extend(['-G', 'Visual Studio 15 2017'])
-
-    elif builder.startswith('linux'):
-        cmake_config_file = "build_files/buildbot/config/blender_linux.cmake"
-        tokens = builder.split("_")
-        glibc = tokens[1]
-        if glibc == 'glibc224':
-            deb_name = "stretch"
-            if builder.endswith('x86_64_cmake'):
-                chroot_name = 'buildbot_' + deb_name + '_x86_64'
-            elif builder.endswith('i686_cmake'):
-                bits = 32
-                chroot_name = 'buildbot_' + deb_name + '_i686'
-            command_prefix = ['schroot', '-c', chroot_name, '--']
-        elif glibc == 'glibc217':
-            command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
-
-    cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file))
-
-    # Prepare CMake options needed to configure cuda binaries compilation, 64bit only.
-    if bits == 64:
-        cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=ON")
-        cmake_options.append("-DCUDA_64_BIT_DEVICE_CODE=ON")
-    else:
-        cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=OFF")
-
-    cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir))
-
-    cmake_options += cmake_extra_options
-
-    # Make sure no garbage remained from the previous run
-    if os.path.isdir(install_dir):
-        shutil.rmtree(install_dir)
-
-    for target in targets:
-        print("Building target %s" % (target))
-        # Construct build directory name based on the target
-        target_build_dir = build_dir
-        target_command_prefix = command_prefix[:]
-        if target != 'blender':
-            target_build_dir += '_' + target
-        target_name = 'install'
-        # Tweaking CMake options to respect the target
-        target_cmake_options = cmake_options[:]
     # Do extra git fetch because not all platform/git/buildbot combinations
     # update the origin remote, causing buildinfo to detect local changes.
-        os.chdir(blender_dir)
+    os.chdir(builder.blender_dir)
+
     print("Fetching remotes")
     command = ['git', 'fetch', '--all']
-        print(command)
+    buildbot_utils.call(builder.command_prefix + command)
-        retcode = subprocess.call(target_command_prefix + command)
+
-        if retcode != 0:
+def clean_directories(builder):
-            sys.exit(retcode)
+    # Make sure no garbage remained from the previous run
+    if os.path.isdir(builder.install_dir):
+        shutil.rmtree(builder.install_dir)
+
     # Make sure build directory exists and enter it
-        if not os.path.isdir(target_build_dir):
+    os.makedirs(builder.build_dir, exist_ok=True)
-            os.mkdir(target_build_dir)
+
-        os.chdir(target_build_dir)
-        # Configure the build
-        print("CMake options:")
-        print(target_cmake_options)
-        if os.path.exists('CMakeCache.txt'):
-            print("Removing CMake cache")
-            os.remove('CMakeCache.txt')
     # Remove buildinfo files to force buildbot to re-generate them.
     for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
-            full_path = os.path.join('source', 'creator', buildinfo)
+        full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo)
         if os.path.exists(full_path):
             print("Removing {}" . format(buildinfo))
             os.remove(full_path)
-        retcode = subprocess.call(target_command_prefix + ['cmake', blender_dir] + target_cmake_options)
-        if retcode != 0:
-            print('Configuration FAILED!')
-            sys.exit(retcode)
 
-        if 'win32' in builder or 'win64' in builder:
+def cmake_configure(builder):
-            command = ['cmake', '--build', '.', '--target', target_name, '--config', 'Release']
+    # CMake configuration
+    os.chdir(builder.build_dir)
+
+    cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt')
+    if os.path.exists(cmake_cache):
+        print("Removing CMake cache")
+        os.remove(cmake_cache)
+
+    print("CMake configure:")
+    cmake_options = get_cmake_options(builder)
+    command = ['cmake', builder.blender_dir] + cmake_options
+    buildbot_utils.call(builder.command_prefix + command)
+
+def cmake_build(builder):
+    # CMake build
+    os.chdir(builder.build_dir)
+
+    if builder.platform == 'win':
+        command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release']
     else:
-            command = ['make', '-s', '-j2', target_name]
+        command = ['make', '-s', '-j2', 'install']
 
-        print("Executing command:")
+    print("CMake build:")
-        print(command)
+    buildbot_utils.call(builder.command_prefix + command)
-        retcode = subprocess.call(target_command_prefix + command)
 
-        if retcode != 0:
+if __name__ == "__main__":
-            sys.exit(retcode)
+    builder = buildbot_utils.create_builder_from_arguments()
-
+    update_git(builder)
-else:
+    clean_directories(builder)
-    print("Unknown building system")
+    cmake_configure(builder)
-    sys.exit(1)
+    cmake_build(builder)

build_files/buildbot/slave_pack.py

@@ -22,48 +22,36 @@
 # system and zipping it into buildbot_upload.zip. This is then uploaded
 # to the master in the next buildbot step.
 
+import buildbot_utils
 import os
-import subprocess
 import sys
-import zipfile
 
-# get builder name
+def get_package_name(builder, platform=None):
-if len(sys.argv) < 2:
+    info = buildbot_utils.VersionInfo(builder)
-    sys.stderr.write("Not enough arguments, expecting builder name\n")
+
+    package_name = 'blender-' + info.version + '-' + info.hash
+    if platform:
+      package_name += '-' + platform
+    if builder.branch != 'master':
+        package_name = builder.branch + "-" + package_name
+
+    return package_name
+
+def create_buildbot_upload_zip(builder, package_filepath, package_filename):
+    import zipfile
+
+    buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
+    if os.path.exists(buildbot_upload_zip):
+        os.remove(buildbot_upload_zip)
+
+    try:
+        z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
+        z.write(package_filepath, arcname=package_filename)
+        z.close()
+    except Exception as ex:
+        sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
         sys.exit(1)
 
-builder = sys.argv[1]
-# Never write branch if it is master.
-branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
-
-blender_dir = os.path.join('..', 'blender.git')
-build_dir = os.path.join('..', 'build', builder)
-install_dir = os.path.join('..', 'install', builder)
-buildbot_upload_zip = os.path.abspath(os.path.join(os.path.dirname(install_dir), "buildbot_upload.zip"))
-
-upload_filename = None  # Name of the archive to be uploaded
-                        # (this is the name of archive which will appear on the
-                        # download page)
-upload_filepath = None  # Filepath to be uploaded to the server
-                        # (this folder will be packed)
-
-
-def parse_header_file(filename, define):
-    import re
-    regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
-    with open(filename, "r") as file:
-        for l in file:
-            match = regex.match(l)
-            if match:
-                return match.group(1)
-    return None
-
-
-# Make sure install directory always exists
-if not os.path.exists(install_dir):
-    os.makedirs(install_dir)
-
-
 def create_tar_bz2(src, dest, package_name):
     # One extra to remove leading os.sep when cleaning root for package_root
     ln = len(src) + 1
@@ -80,163 +68,97 @@ def create_tar_bz2(src, dest, package_name):
         package.add(entry[0], entry[1], recursive=False)
     package.close()
 
+def cleanup_files(dirpath, extension):
+    for f in os.listdir(dirpath):
+        filepath = os.path.join(dirpath, f)
+        if os.path.isfile(filepath) and f.endswith(extension):
+            os.remove(filepath)
 
-if builder.find('cmake') != -1:
+def find_file(dirpath, extension):
-    # CMake
+    for f in os.listdir(dirpath):
-    if 'win' in builder or 'mac' in builder:
+        filepath = os.path.join(dirpath, f)
-        os.chdir(build_dir)
+        if os.path.isfile(filepath) and f.endswith(extension):
+            return f
+    return None
 
-        files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')]
-        for f in files:
-            os.remove(f)
-        retcode = subprocess.call(['cpack', '-G', 'ZIP'])
-        result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
 
-        # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
+def pack_mac(builder):
-        # in the CMake yet. For until then we do some magic here.
+    os.chdir(builder.build_dir)
-        tokens = result_file.split('-')
+    cleanup_files(builder.build_dir, '.zip')
-        blender_version = tokens[1].split('.')
-        blender_full_version = '.'.join(blender_version[0:2])
-        git_hash = tokens[2].split('.')[1]
-        platform = builder.split('_')[0]
-        if platform == 'mac':
-            # Special exception for OSX
-            platform = 'OSX-10.9-'
-            if builder.endswith('x86_64_10_9_cmake'):
-                platform += 'x86_64'
-        if builder.endswith('vc2015'):
-            platform += "-vc14"
-        builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform)
-        # NOTE: Blender 2.7 is already respected by blender_full_version.
-        if branch != '' and branch != 'blender2.7':
-            builderified_name = branch + "-" + builderified_name
 
-        os.rename(result_file, "{}.zip".format(builderified_name))
+    package_name = get_package_name(builder, 'OSX-10.9-x86_64')
-        # create zip file
+    package_filename = package_name + '.zip'
-        try:
-            if os.path.exists(buildbot_upload_zip):
-                os.remove(buildbot_upload_zip)
-            z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
-            z.write("{}.zip".format(builderified_name))
-            z.close()
-            sys.exit(retcode)
-        except Exception as ex:
-            sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
-            sys.exit(1)
 
-    elif builder.startswith('linux_'):
+    buildbot_utils.call(['cpack', '-G', 'ZIP'])
-        blender = os.path.join(install_dir, 'blender')
+    package_filepath = find_file(builder.build_dir, '.zip')
 
-        buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
+    create_buildbot_upload_zip(builder, package_filepath, package_filename)
-        blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
 
-        # Get version information
+
-        blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
+def pack_win(builder):
-        blender_version = "%d.%d" % (blender_version // 100, blender_version % 100)
+    os.chdir(builder.build_dir)
-        blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
+    cleanup_files(builder.build_dir, '.zip')
-        blender_glibc = builder.split('_')[1]
+
-        command_prefix = []
+    package_name = get_package_name(builder, 'win' + str(builder.bits))
-        bits = 64
+    package_filename = package_name + '.zip'
+
+    buildbot_utils.call(['cpack', '-G', 'ZIP'])
+    package_filepath = find_file(builder.build_dir, '.zip')
+
+    create_buildbot_upload_zip(builder, package_filepath, package_filename)
+
+
+def pack_linux(builder):
+    blender_executable = os.path.join(builder.install_dir, 'blender')
+
+    info = buildbot_utils.VersionInfo(builder)
+    blender_glibc = builder.name.split('_')[1]
     blender_arch = 'x86_64'
 
-        if blender_glibc == 'glibc224':
-            if builder.endswith('x86_64_cmake'):
-                chroot_name = 'buildbot_stretch_x86_64'
-            elif builder.endswith('i686_cmake'):
-                chroot_name = 'buildbot_stretch_i686'
-                bits = 32
-                blender_arch = 'i686'
-            command_prefix = ['schroot', '-c', chroot_name, '--']
-        elif blender_glibc == 'glibc217':
-            command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
-
     # Strip all unused symbols from the binaries
     print("Stripping binaries...")
-        subprocess.call(command_prefix + ['strip', '--strip-all', blender])
+    buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
 
     print("Stripping python...")
-        py_target = os.path.join(install_dir, blender_version)
+    py_target = os.path.join(builder.install_dir, info.version)
-        subprocess.call(command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
+    buildbot_utils.call(builder.command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
 
     # Copy all specific files which are too specific to be copied by
     # the CMake rules themselves
     print("Copying extra scripts and libs...")
 
     extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
-        mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
+    mesalibs = os.path.join(extra, 'mesalibs' + str(builder.bits) + '.tar.bz2')
-        software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
+    software_gl = os.path.join(builder.blender_dir, 'release', 'bin', 'blender-softwaregl')
-        icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
+    icons = os.path.join(builder.blender_dir, 'release', 'freedesktop', 'icons')
 
-        os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
+    os.system('tar -xpf %s -C %s' % (mesalibs, builder.install_dir))
-        os.system('cp %s %s' % (software_gl, install_dir))
+    os.system('cp %s %s' % (software_gl, builder.install_dir))
-        os.system('cp -r %s %s' % (icons, install_dir))
+    os.system('cp -r %s %s' % (icons, builder.install_dir))
-        os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
+    os.system('chmod 755 %s' % (os.path.join(builder.install_dir, 'blender-softwaregl')))
 
-        # Construct archive name
+    # Construct package name
-        package_name = 'blender-%s-%s-linux-%s-%s' % (blender_version,
+    platform_name = 'linux-' + blender_glibc + '-' + blender_arch
-                                                      blender_hash,
+    package_name = get_package_name(builder, platform_name)
-                                                      blender_glibc,
+    package_filename = package_name + ".tar.bz2"
-                                                      blender_arch)
-        # NOTE: Blender 2.7 is already respected by blender_full_version.
-        if branch != '' and branch != 'blender2.7':
-            package_name = branch + "-" + package_name
-
-        upload_filename = package_name + ".tar.bz2"
 
     print("Creating .tar.bz2 archive")
-        upload_filepath = install_dir + '.tar.bz2'
+    package_filepath = builder.install_dir + '.tar.bz2'
-        create_tar_bz2(install_dir, upload_filepath, package_name)
+    create_tar_bz2(builder.install_dir, package_filepath, package_name)
-else:
+
-    print("Unknown building system")
+    # Create buildbot_upload.zip
-    sys.exit(1)
+    create_buildbot_upload_zip(builder, package_filepath, package_filename)
 
 
-if upload_filepath is None:
+if __name__ == "__main__":
-    # clean release directory if it already exists
+    builder = buildbot_utils.create_builder_from_arguments()
-    release_dir = 'release'
 
-    if os.path.exists(release_dir):
+    # Make sure install directory always exists
-        for f in os.listdir(release_dir):
+    os.makedirs(builder.install_dir, exist_ok=True)
-            if os.path.isfile(os.path.join(release_dir, f)):
-                os.remove(os.path.join(release_dir, f))
 
-    # create release package
+    if builder.platform == 'mac':
-    try:
+        pack_mac(builder)
-        subprocess.call(['make', 'package_archive'])
+    elif builder.platform == 'win':
-    except Exception as ex:
+        pack_win(builder)
-        sys.stderr.write('Make package release failed' + str(ex) + '\n')
+    elif builder.platform == 'linux':
-        sys.exit(1)
+        pack_linux(builder)
-
-    # find release directory, must exist this time
-    if not os.path.exists(release_dir):
-        sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
-        sys.exit(1)
-
-    # find release package
-    file = None
-    filepath = None
-
-    for f in os.listdir(release_dir):
-        rf = os.path.join(release_dir, f)
-        if os.path.isfile(rf) and f.startswith('blender'):
-            file = f
-            filepath = rf
-
-    if not file:
-        sys.stderr.write("Failed to find release package.\n")
-        sys.exit(1)
-
-    upload_filename = file
-    upload_filepath = filepath
-
-# create zip file
-try:
-    upload_zip = os.path.join(buildbot_upload_zip)
-    if os.path.exists(upload_zip):
-        os.remove(upload_zip)
-    z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
-    z.write(upload_filepath, arcname=upload_filename)
-    z.close()
-except Exception as ex:
-    sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
-    sys.exit(1)

build_files/buildbot/slave_rsync.py

@@ -21,23 +21,17 @@
 # Runs on buildbot slave, rsync zip directly to buildbot server rather
 # than using upload which is much slower
 
+import buildbot_utils
 import os
 import sys
 
-# get builder name
+if __name__ == "__main__":
-if len(sys.argv) < 2:
+    builder = buildbot_utils.create_builder_from_arguments()
-    sys.stderr.write("Not enough arguments, expecting builder name\n")
-    sys.exit(1)
 
-builder = sys.argv[1]
+    # rsync, this assumes ssh keys are setup so no password is needed
+    local_zip = "buildbot_upload.zip"
+    remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/"
+    remote_zip = remote_folder + "buildbot_upload_" + builder.name + ".zip"
 
-# rsync, this assumes ssh keys are setup so no password is needed
+    command = ["rsync", "-avz", local_zip, remote_zip]
-local_zip = "buildbot_upload.zip"
+    buildbot_utils.call(command)
-remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/"
-remote_zip = remote_folder + "buildbot_upload_" + builder + ".zip"
-command = "rsync -avz %s %s" % (local_zip, remote_zip)
-
-print(command)
-
-ret = os.system(command)
-sys.exit(ret)

build_files/buildbot/slave_test.py

@@ -18,59 +18,39 @@
 
 # <pep8 compliant>
 
-import subprocess
+import buildbot_utils
 import os
 import sys
 
-# get builder name
+def get_ctest_environment(builder):
-if len(sys.argv) < 2:
+    info = buildbot_utils.VersionInfo(builder)
-    sys.stderr.write("Not enough arguments, expecting builder name\n")
+    blender_version_dir = os.path.join(builder.install_dir, info.version)
-    sys.exit(1)
 
-builder = sys.argv[1]
+    env = os.environ.copy()
+    env['BLENDER_SYSTEM_SCRIPTS'] = os.path.join(blender_version_dir, 'scripts')
+    env['BLENDER_SYSTEM_DATAFILES'] = os.path.join(blender_version_dir, 'datafiles')
+    return env
 
-# we run from build/ directory
+def get_ctest_arguments(builder):
-blender_dir = '../blender.git'
+    args = ['--output-on-failure']
+    if builder.platform == 'win':
+        args += ['-C', 'Release']
+    return args
 
-if "cmake" in builder:
+def test(builder):
+    os.chdir(builder.build_dir)
+
+    command = builder.command_prefix  + ['ctest'] + get_ctest_arguments(builder)
+    ctest_env = get_ctest_environment(builder)
+    buildbot_utils.call(command, env=ctest_env, exit_on_error=False)
+
+if __name__ == "__main__":
     print("Automated tests are still DISABLED!")
     sys.exit(0)
 
-    build_dir = os.path.abspath(os.path.join('..', 'build', builder))
+    builder = buildbot_utils.create_builder_from_arguments()
-    install_dir = os.path.abspath(os.path.join('..', 'install', builder))
+    test(builder)
-    # NOTE: For quick test only to see if the approach work.
-    # n the future must be replaced with an actual blender version.
-    blender_version = '2.80'
-    blender_version_dir = os.path.join(install_dir, blender_version)
-    command_prefix = []
-    extra_ctest_args = []
-
-    if builder.startswith('win'):
-        extra_ctest_args += ['-C', 'Release']
-    elif builder.startswith('linux'):
-        tokens = builder.split("_")
-        glibc = tokens[1]
-        if glibc == 'glibc224':
-            deb_name = "stretch"
-            if builder.endswith('x86_64_cmake'):
-                chroot_name = 'buildbot_' + deb_name + '_x86_64'
-            elif builder.endswith('i686_cmake'):
-                chroot_name = 'buildbot_' + deb_name + '_i686'
-            command_prefix = ['schroot', '--preserve-environment', '-c', chroot_name, '--']
-        elif glibc == 'glibc217':
-            command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
-
-    ctest_env = os.environ.copy()
-    ctest_env['BLENDER_SYSTEM_SCRIPTS'] = os.path.join(blender_version_dir, 'scripts')
-    ctest_env['BLENDER_SYSTEM_DATAFILES'] = os.path.join(blender_version_dir, 'datafiles')
-
-    os.chdir(build_dir)
-    retcode = subprocess.call(command_prefix + ['ctest', '--output-on-failure'] + extra_ctest_args,
-                              env=ctest_env)
 
     # Always exit with a success, for until we know all the tests are passing
     # on all builders.
     sys.exit(0)
-else:
-    print("Unknown building system")
-    sys.exit(1)

build_files/buildbot/slave_update.py

@@ -18,14 +18,13 @@
 
 # <pep8 compliant>
 
+import buildbot_utils
 import os
-import runpy
 
-# We run from build/ directory.
+if __name__ == "__main__":
-blender_dir = os.path.join('..', 'blender.git')
+    builder = buildbot_utils.create_builder_from_arguments()
-blender_dir = os.path.abspath(blender_dir)
+    os.chdir(builder.blender_dir)
-os.chdir(blender_dir)
 
-# Run make update which handles all libraries and submodules.
+    # Run make update which handles all libraries and submodules.
-make_update = os.path.join(blender_dir, "build_files", "utils", "make_update.py")
+    make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py")
-runpy.run_path(make_update)
+    buildbot_utils.call(['python3', make_update])