archive/blender-buildbot
Archived
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Support multiple packages in single upload, refactor unpack script

Browse Source
This commit is contained in:
Brecht Van Lommel
2019-09-02 11:04:39 +02:00
parent 8349e52e4a
commit 50e508fd18
1 changed files with 134 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
master_unpack.py
View File

@@ -21,130 +21,168 @@
# <pep8 compliant> # <pep8 compliant>
import argparse
import os import os
import shutil import shutil
import sys import sys
import zipfile import zipfile
# Parse package filename to extract branch and platform
class Package:
def __init__(self, zipname):
self.zipname = zipname
self.filename = os.path.basename(zipname)
self.platform = self._get_platform(self.filename)
self.branch = self._get_branch(self.filename)
# extension stripping if self.platform == '':
def strip_extension(filename): sys.stderr.write('Failed to detect platform ' +
extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe' 'from package: %r\n' % self.filename)
sys.exit(1)
for ext in extensions: # extension stripping
if filename.endswith(ext): def _strip_extension(self, filename):
filename = filename[:-len(ext)] extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
return filename for ext in extensions:
if filename.endswith(ext):
filename = filename[:-len(ext)]
return filename
# extract platform from package name
def _get_platform(self, filename):
# name is blender-version-platform.extension. we want to get the
# platform out, but there may be some variations, so we fiddle a
# bit to handle current and hopefully future names
filename = self._strip_extension(filename)
filename = self._strip_extension(filename)
tokens = filename.split("-")
platforms = ('osx', 'mac', 'bsd',
'win', 'linux', 'source',
'irix', 'solaris', 'mingw')
platform_tokens = []
found = False
for i, token in enumerate(tokens):
if not found:
for platform in platforms:
if platform in token.lower():
found = True
break
if found:
platform_tokens += [token]
return '-'.join(platform_tokens)
# extract platform from package name def _get_branch(self, filename):
def get_platform(filename): tokens = filename.split("-")
# name is blender-version-platform.extension. we want to get the branch = ""
# platform out, but there may be some variations, so we fiddle a
# bit to handle current and hopefully future names
filename = strip_extension(filename)
filename = strip_extension(filename)
tokens = filename.split("-") for token in tokens:
platforms = ('osx', 'mac', 'bsd', if token == "blender":
'win', 'linux', 'source', return branch
'irix', 'solaris', 'mingw')
platform_tokens = []
found = False
for i, token in enumerate(tokens): if branch == "":
if not found: branch = token
for platform in platforms: else:
if platform in token.lower(): branch = branch + "-" + token
found = True
break
if found: return ""
platform_tokens += [token]
return '-'.join(platform_tokens)
def get_branch(filename): def get_download_dir(branch):
tokens = filename.split("-") download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
branch = "" if not branch or branch == 'master':
directory = download_prefix
elif branch == 'experimental-build':
directory = os.path.join(download_prefix, "experimental")
else:
directory = os.path.join(download_prefix, "branches", branch)
os.makedirs(directory, exist_ok=True)
for token in tokens: return directory
if token == "blender":
return branch
if branch == "": def open_zipfile(filename):
branch = token # Open zip file
else: if not os.path.exists(filename):
branch = branch + "-" + token sys.stderr.write("File %r not found.\n" % filename)
sys.exit(1)
return "" try:
return zipfile.ZipFile(filename, "r")
except Exception as ex:
sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
sys.exit(1)
# get filename def get_zipfile_packages(zipfile):
if len(sys.argv) < 2: # List packages in zip file
sys.stderr.write("Not enough arguments, expecting file to unpack\n") packages = [Package(zipname) for zipname in zipfile.namelist()]
sys.exit(1) if len(packages) == 0:
sys.stderr.write('Empty zip file\n')
sys.exit(1)
return packages
filename = sys.argv[1] def get_branch_platform(packages):
# Extract branch and platform names
branch = packages[0].branch
platform = packages[0].platform
for package in packages:
if package.branch != branch or package.platform != platform:
sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
sys.exit(1)
# open zip file return branch, platform
if not os.path.exists(filename):
sys.stderr.write("File %r not found.\n" % filename)
sys.exit(1)
try: def extract_zipfile_packages(zipfile, packages, branch):
z = zipfile.ZipFile(filename, "r") # Extract packages from zip file
except Exception as ex: directory = get_download_dir(branch)
sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
sys.exit(1)
if len(z.namelist()) != 1: for package in packages:
sys.stderr.write("Expected one file in %r." % filename) filepath = os.path.join(directory, package.filename)
sys.exit(1)
package = z.namelist()[0] try:
packagename = os.path.basename(package) zf = zipfile.open(package.zipname)
f = open(filepath, "wb")
# detect platform and branch shutil.copyfileobj(zf, f)
platform = get_platform(packagename) os.chmod(filepath, 0o644)
branch = get_branch(packagename)
if platform == '': zf.close()
sys.stderr.write('Failed to detect platform ' + except Exception as ex:
'from package: %r\n' % packagename) sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
sys.exit(1) sys.exit(1)
# extract def remove_older_packages(branch, platform, new_packages):
download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/" # Remove other files from the same platform and branch
if not branch or branch == 'master': directory = get_download_dir(branch)
directory = download_prefix
elif branch == 'experimental-build':
directory = os.path.join(download_prefix, "experimental")
else:
directory = os.path.join(download_prefix, "branches", branch)
os.makedirs(directory, exist_ok=True)
try: for filename in os.listdir(directory):
filename = os.path.join(directory, packagename) package = Package(filename)
zf = z.open(package) if package.platform == platform and package.branch == branch:
f = open(filename, "wb") is_new_package = False
for new_package in new_packages:
if package.filename == new_package.filename:
is_new_package = True
shutil.copyfileobj(zf, f) if not is_new_package:
os.chmod(filename, 0o644) try:
os.remove(os.path.join(directory, filename))
except Exception as ex:
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
zf.close()
z.close()
except Exception as ex:
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
sys.exit(1)
# remove other files from the same platform and branch if __name__ == "__main__":
try: parser = argparse.ArgumentParser()
for f in os.listdir(directory): parser.add_argument('filename')
if get_platform(f) == platform and get_branch(f) == branch: args = parser.parse_args()
if f != packagename:
os.remove(os.path.join(directory, f)) with open_zipfile(args.filename) as zipfile:
except Exception as ex: packages = get_zipfile_packages(zipfile)
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex)) branch, platform = get_branch_platform(packages)
sys.exit(1) extract_zipfile_packages(zipfile, packages, branch)
remove_older_packages(branch, platform, packages)