blender-buildbot/master_unpack.py

# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

# Runs on Buildbot master, to unpack incoming unload.zip into latest
# builds directory and remove older builds.

# <pep8 compliant>

import argparse
import os
import shutil
import sys
import time
import zipfile

DOWNLOAD_DIR = "/data/www/vhosts/builder.blender.org/webroot/download/"

# Parse package filename to extract branch and platform
class Package:
    def __init__(self, zipname):
        self.zipname = zipname
        self.filename = os.path.basename(zipname)
        self.version = ""
        self.branch = ""
        self.platform = ""
        self.parsed = False
        self.parse_filename()

    # extension stripping
    def _strip_extension(self, filename):
        filename, _ = os.path.splitext(filename)
        if filename.endswith('.tar'):
            filename, _ = os.path.splitext(filename)

        return filename

    # extract platform from package name
    def parse_filename(self):
        # Name is one one of:
        # blender-version-platform: official release
        # blender-version-hash-platform: daily build
        # branch-blender-version-hash-platform: branch build
        filename = self._strip_extension(self.filename)
        tokens = filename.split("-")

        if len(tokens) < 3:
            return
        if "blender" not in tokens:
            return

        blender_index = tokens.index("blender")
        if not (blender_index + 2 < len(tokens)):
            return

        # Detect branch.
        branch = "-".join(tokens[:blender_index])

        # Detect version.
        if branch == "":
            version = tokens[blender_index + 1]
            if len(tokens) == 3:
                version += " release"
        else:
            version = ""

        # Detect platform.
        platform = tokens[-1]
        platforms = ('osx', 'mac', 'bsd',
                     'win', 'linux', 'source',
                     'irix', 'solaris', 'mingw')
        platform_found = False
        for name in platforms:
            if platform.lower().startswith(name):
                platform_found = True
        if not platform_found:
            return

        self.version = version
        self.platform = platform
        self.branch = branch
        self.parsed = True

def get_download_dir(branch):
    if not branch or branch == 'master':
        directory = DOWNLOAD_DIR
    else:
        directory = os.path.join(DOWNLOAD_DIR, branch)
        os.makedirs(directory, exist_ok=True)
        os.chmod(directory, 0o755)

    return directory

def open_zipfile(filename):
    # Open zip file
    if not os.path.exists(filename):
        sys.stderr.write("File %r not found.\n" % filename)
        sys.exit(1)

    try:
        return zipfile.ZipFile(filename, "r")
    except Exception as ex:
        sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
        sys.exit(1)

def get_zipfile_packages(zipfile):
    # List packages in zip file
    packages = [Package(zipname) for zipname in zipfile.namelist()]
    for package in packages:
        if not package.parsed:
            sys.stderr.write("Failed to parse package filename: %s\n" % str(package.filename))
            sys.exit(1)
    if len(packages) == 0:
        sys.stderr.write('Empty zip file\n')
        sys.exit(1)
    return packages

def get_branch_platform(packages):
    # Extract branch and platform names
    branch = packages[0].branch
    platform = packages[0].platform
    version = packages[0].version

    if platform == '':
        sys.stderr.write('Failed to detect platform ' +
            'from package: %r\n' % packages[0].filename)
        sys.exit(1)

    for package in packages:
        if package.branch != branch or package.platform != platform or package.version != version:
            sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
            sys.exit(1)

    return branch, platform, version

def extract_zipfile_packages(zipfile, packages, branch):
    # Extract packages from zip file
    directory = get_download_dir(branch)

    for package in packages:
        filepath = os.path.join(directory, package.filename)

        try:
            zf = zipfile.open(package.zipname)
            f = open(filepath, "wb")

            shutil.copyfileobj(zf, f)
            os.chmod(filepath, 0o644)

            zf.close()
        except Exception as ex:
            sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
            sys.exit(1)

def remove_replaced_packages(branch, platform, version, new_packages):
    # Remove other files from the same platform and branch that are replaced
    # by the new packages.
    directory = get_download_dir(branch)

    for filename in os.listdir(directory):
        package = Package(filename)
        if package.platform == platform and package.branch == branch and package.version == version:
            is_new_package = False
            for new_package in new_packages:
                if package.filename == new_package.filename:
                    is_new_package = True

            if not is_new_package:
                try:
                    print("Removing older package version", filename)
                    os.remove(os.path.join(directory, filename))
                except Exception as ex:
                    sys.stderr.write('Failed to remove replaced package: %s\n' % str(ex))

def remove_old_packages():
    # Remove any branch packages that are 100 days or older.
    cutoff_time = time.time() - 100 * 86400

    for dirname in os.listdir(DOWNLOAD_DIR):
        dirpath = os.path.join(DOWNLOAD_DIR, dirname)
        if not os.path.isdir(dirpath):
            continue

        for filename in os.listdir(dirpath):
            filepath = os.path.join(dirpath, filename)
            if not os.path.isfile(filepath):
                continue

            file_mtime = os.stat(filepath).st_mtime
            if file_mtime < cutoff_time:
                try:
                    print("Removing old branch build", filename)
                    os.remove(filepath)
                except Exception as ex:
                    sys.stderr.write('Failed to remove old package: %s\n' % str(ex))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('filename')
    args = parser.parse_args()

    with open_zipfile(args.filename) as zipfile:
        packages = get_zipfile_packages(zipfile)
        branch, platform, version = get_branch_platform(packages)
        extract_zipfile_packages(zipfile, packages, branch)
        remove_replaced_packages(branch, platform, version, packages)

    remove_old_packages()