# ##### BEGIN GPL LICENSE BLOCK ##### # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # ##### END GPL LICENSE BLOCK ##### # Runs on Buildbot master, to unpack incoming unload.zip into latest # builds directory and remove older builds. # import argparse import os import shutil import sys import time import zipfile DOWNLOAD_DIR = "/data/www/vhosts/builder.blender.org/webroot/download/" # Parse package filename to extract branch and platform class Package: def __init__(self, zipname): self.zipname = zipname self.filename = os.path.basename(zipname) self.version = "" self.branch = "" self.platform = "" self.parsed = False self.parse_filename() # extension stripping def _strip_extension(self, filename): filename, _ = os.path.splitext(filename) if filename.endswith('.tar'): filename, _ = os.path.splitext(filename) return filename # extract platform from package name def parse_filename(self): # Name is one one of: # blender-version-platform: official release # blender-version-hash-platform: daily build # branch-blender-version-hash-platform: branch build filename = self._strip_extension(self.filename) tokens = filename.split("-") if len(tokens) < 3: return if "blender" not in tokens: return blender_index = tokens.index("blender") if not (blender_index + 2 < len(tokens)): return # Detect branch. branch = "-".join(tokens[:blender_index]) # Detect version. if branch == "": version = tokens[blender_index + 1] if len(tokens) == 3: version += " release" else: version = "" # Detect platform. platform = tokens[-1] platforms = ('osx', 'mac', 'bsd', 'win', 'linux', 'source', 'irix', 'solaris', 'mingw') platform_found = False for name in platforms: if platform.lower().startswith(name): platform_found = True if not platform_found: return self.version = version self.platform = platform self.branch = branch self.parsed = True def get_download_dir(branch): if not branch or branch == 'master': directory = DOWNLOAD_DIR elif branch == 'experimental-build': directory = os.path.join(DOWNLOAD_DIR, "experimental") else: directory = os.path.join(DOWNLOAD_DIR, branch) os.makedirs(directory, exist_ok=True) os.chmod(directory, 0o755) return directory def open_zipfile(filename): # Open zip file if not os.path.exists(filename): sys.stderr.write("File %r not found.\n" % filename) sys.exit(1) try: return zipfile.ZipFile(filename, "r") except Exception as ex: sys.stderr.write('Failed to open zip file: %s\n' % str(ex)) sys.exit(1) def get_zipfile_packages(zipfile): # List packages in zip file packages = [Package(zipname) for zipname in zipfile.namelist()] for package in packages: if not package.parsed: sys.stderr.write("Failed to parse package filename: %s\n" % str(package.filename)) sys.exit(1) if len(packages) == 0: sys.stderr.write('Empty zip file\n') sys.exit(1) return packages def get_branch_platform(packages): # Extract branch and platform names branch = packages[0].branch platform = packages[0].platform version = packages[0].version if platform == '': sys.stderr.write('Failed to detect platform ' + 'from package: %r\n' % packages[0].filename) sys.exit(1) for package in packages: if package.branch != branch or package.platform != platform or package.version != version: sys.stderr.write('All packages in the zip file must have the same branch and platform\n') sys.exit(1) return branch, platform, version def extract_zipfile_packages(zipfile, packages, branch): # Extract packages from zip file directory = get_download_dir(branch) for package in packages: filepath = os.path.join(directory, package.filename) try: zf = zipfile.open(package.zipname) f = open(filepath, "wb") shutil.copyfileobj(zf, f) os.chmod(filepath, 0o644) zf.close() except Exception as ex: sys.stderr.write('Failed to unzip package: %s\n' % str(ex)) sys.exit(1) def remove_replaced_packages(branch, platform, version, new_packages): # Remove other files from the same platform and branch that are replaced # by the new packages. directory = get_download_dir(branch) for filename in os.listdir(directory): package = Package(filename) if package.platform == platform and package.branch == branch and package.version == version: is_new_package = False for new_package in new_packages: if package.filename == new_package.filename: is_new_package = True if not is_new_package: try: print("Removing older package version", filename) os.remove(os.path.join(directory, filename)) except Exception as ex: sys.stderr.write('Failed to remove replaced package: %s\n' % str(ex)) def remove_old_packages(): # Remove any branch packages that are 100 days or older. cutoff_time = time.time() - 100 * 86400 for dirname in os.listdir(DOWNLOAD_DIR): dirpath = os.path.join(DOWNLOAD_DIR, dirname) if not os.path.isdir(dirpath): continue for filename in os.listdir(dirpath): filepath = os.path.join(dirpath, filename) if not os.path.isfile(filepath): continue file_mtime = os.stat(filepath).st_mtime if file_mtime < cutoff_time: try: print("Removing old branch build", filename) os.remove(filepath) except Exception as ex: sys.stderr.write('Failed to remove old package: %s\n' % str(ex)) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('filename') args = parser.parse_args() with open_zipfile(args.filename) as zipfile: packages = get_zipfile_packages(zipfile) branch, platform, version = get_branch_platform(packages) extract_zipfile_packages(zipfile, packages, branch) remove_replaced_packages(branch, platform, version, packages) remove_old_packages()