blender-buildbot/master_unpack.py

# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

# Runs on Buildbot master, to unpack incoming unload.zip into latest
# builds directory and remove older builds.

# <pep8 compliant>

import argparse
import os
import shutil
import sys
import time
import zipfile

DOWNLOAD_DIR = "/data/www/vhosts/builder.blender.org/webroot/download/"

# Parse package filename to extract branch and platform
class Package:
    def __init__(self, zipname):
        self.zipname = zipname
        self.filename = os.path.basename(zipname)
        self.platform = self._get_platform(self.filename)
        self.branch = self._get_branch(self.filename)
        self.version = self._get_version(self.filename)

    # extension stripping
    def _strip_extension(self, filename):
        extensions = '.zip', '.tar', '.bz2', '.xz', '.gz', '.tgz', '.tbz', '.exe'

        for ext in extensions:
            if filename.endswith(ext):
                filename = filename[:-len(ext)]

        return filename

    # extract platform from package name
    def _get_platform(self, filename):
        # name is blender-version-platform.extension. we want to get the
        # platform out, but there may be some variations, so we fiddle a
        # bit to handle current and hopefully future names
        filename = self._strip_extension(filename)
        filename = self._strip_extension(filename)

        tokens = filename.split("-")
        platforms = ('osx', 'mac', 'bsd',
                     'win', 'linux', 'source',
                     'irix', 'solaris', 'mingw')
        platform_tokens = []
        found = False

        for i, token in enumerate(tokens):
            if not found:
                for platform in platforms:
                    if platform in token.lower():
                        found = True
                        break

            if found:
                platform_tokens += [token]

        return '-'.join(platform_tokens)


    def _get_version(self, filename):
        tokens = filename.split("-")

        use_next_token = False
        for token in tokens:
            if use_next_token:
                return token
            elif token == "blender":
                use_next_token = True

        return ""

    def _get_branch(self, filename):
        tokens = filename.split("-")
        branch = ""

        for token in tokens:
            if token == "blender":
                return branch

            if branch == "":
                branch = token
            else:
                branch = branch + "-" + token

        return ""


def get_download_dir(branch):
    if not branch or branch == 'master':
        directory = DOWNLOAD_DIR
    elif branch == 'experimental-build':
        directory = os.path.join(DOWNLOAD_DIR, "experimental")
    else:
        directory = os.path.join(DOWNLOAD_DIR, branch)
        os.makedirs(directory, exist_ok=True)
        os.chmod(directory, 0o755)

    return directory

def open_zipfile(filename):
    # Open zip file
    if not os.path.exists(filename):
        sys.stderr.write("File %r not found.\n" % filename)
        sys.exit(1)

    try:
        return zipfile.ZipFile(filename, "r")
    except Exception as ex:
        sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
        sys.exit(1)

def get_zipfile_packages(zipfile):
    # List packages in zip file
    packages = [Package(zipname) for zipname in zipfile.namelist()]
    if len(packages) == 0:
        sys.stderr.write('Empty zip file\n')
        sys.exit(1)
    return packages

def get_branch_platform(packages):
    # Extract branch and platform names
    branch = packages[0].branch
    platform = packages[0].platform
    version = packages[0].version

    if platform == '':
        sys.stderr.write('Failed to detect platform ' +
            'from package: %r\n' % packages[0].filename)
        sys.exit(1)

    for package in packages:
        if package.branch != branch or package.platform != platform or package.version != version:
            sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
            sys.exit(1)

    return branch, platform, version

def extract_zipfile_packages(zipfile, packages, branch):
    # Extract packages from zip file
    directory = get_download_dir(branch)

    for package in packages:
        filepath = os.path.join(directory, package.filename)

        try:
            zf = zipfile.open(package.zipname)
            f = open(filepath, "wb")

            shutil.copyfileobj(zf, f)
            os.chmod(filepath, 0o644)

            zf.close()
        except Exception as ex:
            sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
            sys.exit(1)

def remove_replaced_packages(branch, platform, version, new_packages):
    # Remove other files from the same platform and branch that are replaced
    # by the new packages.
    directory = get_download_dir(branch)

    for filename in os.listdir(directory):
        package = Package(filename)
        if package.platform == platform and package.branch == branch and package.version == version:
            is_new_package = False
            for new_package in new_packages:
                if package.filename == new_package.filename:
                    is_new_package = True

            if not is_new_package:
                try:
                    print("Removing older package version", filename)
                    os.remove(os.path.join(directory, filename))
                except Exception as ex:
                    sys.stderr.write('Failed to remove replaced package: %s\n' % str(ex))

def remove_old_packages():
    # Remove any branch packages that are 100 days or older.
    cutoff_time = time.time() - 100 * 86400

    for dirname in os.listdir(DOWNLOAD_DIR):
        dirpath = os.path.join(DOWNLOAD_DIR, dirname)
        if not os.path.isdir(dirpath):
            continue

        for filename in os.listdir(dirpath):
            filepath = os.path.join(dirpath, filename)
            if not os.path.isfile(filepath):
                continue

            file_mtime = os.stat(filepath).st_mtime
            if file_mtime < cutoff_time:
                try:
                    print("Removing old branch build", filename)
                    os.remove(filepath)
                except Exception as ex:
                    sys.stderr.write('Failed to remove old package: %s\n' % str(ex))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('filename')
    args = parser.parse_args()

    with open_zipfile(args.filename) as zipfile:
        packages = get_zipfile_packages(zipfile)
        branch, platform, version = get_branch_platform(packages)
        extract_zipfile_packages(zipfile, packages, branch)
        remove_replaced_packages(branch, platform, version, packages)

    remove_old_packages()