Support multiple packages in single upload, refactor unpack script
This commit is contained in:
231
master_unpack.py
231
master_unpack.py
@@ -21,130 +21,169 @@
|
|||||||
|
|
||||||
# <pep8 compliant>
|
# <pep8 compliant>
|
||||||
|
|
||||||
|
import argparse
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
|
# Parse package filename to extract branch and platform
|
||||||
|
class Package:
|
||||||
|
def __init__(self, zipname):
|
||||||
|
self.zipname = zipname
|
||||||
|
self.filename = os.path.basename(zipname)
|
||||||
|
self.platform = self._get_platform(self.filename)
|
||||||
|
self.branch = self._get_branch(self.filename)
|
||||||
|
|
||||||
# extension stripping
|
# extension stripping
|
||||||
def strip_extension(filename):
|
def _strip_extension(self, filename):
|
||||||
extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
|
extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
|
||||||
|
|
||||||
for ext in extensions:
|
for ext in extensions:
|
||||||
if filename.endswith(ext):
|
if filename.endswith(ext):
|
||||||
filename = filename[:-len(ext)]
|
filename = filename[:-len(ext)]
|
||||||
|
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
|
# extract platform from package name
|
||||||
|
def _get_platform(self, filename):
|
||||||
|
# name is blender-version-platform.extension. we want to get the
|
||||||
|
# platform out, but there may be some variations, so we fiddle a
|
||||||
|
# bit to handle current and hopefully future names
|
||||||
|
filename = self._strip_extension(filename)
|
||||||
|
filename = self._strip_extension(filename)
|
||||||
|
|
||||||
|
tokens = filename.split("-")
|
||||||
|
platforms = ('osx', 'mac', 'bsd',
|
||||||
|
'win', 'linux', 'source',
|
||||||
|
'irix', 'solaris', 'mingw')
|
||||||
|
platform_tokens = []
|
||||||
|
found = False
|
||||||
|
|
||||||
|
for i, token in enumerate(tokens):
|
||||||
|
if not found:
|
||||||
|
for platform in platforms:
|
||||||
|
if platform in token.lower():
|
||||||
|
found = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if found:
|
||||||
|
platform_tokens += [token]
|
||||||
|
|
||||||
|
return '-'.join(platform_tokens)
|
||||||
|
|
||||||
|
|
||||||
# extract platform from package name
|
def _get_branch(self, filename):
|
||||||
def get_platform(filename):
|
tokens = filename.split("-")
|
||||||
# name is blender-version-platform.extension. we want to get the
|
branch = ""
|
||||||
# platform out, but there may be some variations, so we fiddle a
|
|
||||||
# bit to handle current and hopefully future names
|
|
||||||
filename = strip_extension(filename)
|
|
||||||
filename = strip_extension(filename)
|
|
||||||
|
|
||||||
tokens = filename.split("-")
|
for token in tokens:
|
||||||
platforms = ('osx', 'mac', 'bsd',
|
if token == "blender":
|
||||||
'win', 'linux', 'source',
|
return branch
|
||||||
'irix', 'solaris', 'mingw')
|
|
||||||
platform_tokens = []
|
|
||||||
found = False
|
|
||||||
|
|
||||||
for i, token in enumerate(tokens):
|
if branch == "":
|
||||||
if not found:
|
branch = token
|
||||||
for platform in platforms:
|
else:
|
||||||
if platform in token.lower():
|
branch = branch + "-" + token
|
||||||
found = True
|
|
||||||
break
|
|
||||||
|
|
||||||
if found:
|
return ""
|
||||||
platform_tokens += [token]
|
|
||||||
|
|
||||||
return '-'.join(platform_tokens)
|
|
||||||
|
|
||||||
|
|
||||||
def get_branch(filename):
|
def get_download_dir(branch):
|
||||||
tokens = filename.split("-")
|
download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
|
||||||
branch = ""
|
if not branch or branch == 'master':
|
||||||
|
directory = download_prefix
|
||||||
|
elif branch == 'experimental-build':
|
||||||
|
directory = os.path.join(download_prefix, "experimental")
|
||||||
|
else:
|
||||||
|
directory = os.path.join(download_prefix, branch)
|
||||||
|
os.makedirs(directory, exist_ok=True)
|
||||||
|
|
||||||
for token in tokens:
|
return directory
|
||||||
if token == "blender":
|
|
||||||
return branch
|
|
||||||
|
|
||||||
if branch == "":
|
def open_zipfile(filename):
|
||||||
branch = token
|
# Open zip file
|
||||||
else:
|
if not os.path.exists(filename):
|
||||||
branch = branch + "-" + token
|
sys.stderr.write("File %r not found.\n" % filename)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
return ""
|
try:
|
||||||
|
return zipfile.ZipFile(filename, "r")
|
||||||
|
except Exception as ex:
|
||||||
|
sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
# get filename
|
def get_zipfile_packages(zipfile):
|
||||||
if len(sys.argv) < 2:
|
# List packages in zip file
|
||||||
sys.stderr.write("Not enough arguments, expecting file to unpack\n")
|
packages = [Package(zipname) for zipname in zipfile.namelist()]
|
||||||
sys.exit(1)
|
if len(packages) == 0:
|
||||||
|
sys.stderr.write('Empty zip file\n')
|
||||||
|
sys.exit(1)
|
||||||
|
return packages
|
||||||
|
|
||||||
filename = sys.argv[1]
|
def get_branch_platform(packages):
|
||||||
|
# Extract branch and platform names
|
||||||
|
branch = packages[0].branch
|
||||||
|
platform = packages[0].platform
|
||||||
|
|
||||||
# open zip file
|
if platform == '':
|
||||||
if not os.path.exists(filename):
|
sys.stderr.write('Failed to detect platform ' +
|
||||||
sys.stderr.write("File %r not found.\n" % filename)
|
'from package: %r\n' % packages[0].filename)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
try:
|
for package in packages:
|
||||||
z = zipfile.ZipFile(filename, "r")
|
if package.branch != branch or package.platform != platform:
|
||||||
except Exception as ex:
|
sys.stderr.write('All packages in the zip file must have the same branch and platform\n')
|
||||||
sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
|
sys.exit(1)
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if len(z.namelist()) != 1:
|
return branch, platform
|
||||||
sys.stderr.write("Expected one file in %r." % filename)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
package = z.namelist()[0]
|
def extract_zipfile_packages(zipfile, packages, branch):
|
||||||
packagename = os.path.basename(package)
|
# Extract packages from zip file
|
||||||
|
directory = get_download_dir(branch)
|
||||||
|
|
||||||
# detect platform and branch
|
for package in packages:
|
||||||
platform = get_platform(packagename)
|
filepath = os.path.join(directory, package.filename)
|
||||||
branch = get_branch(packagename)
|
|
||||||
|
|
||||||
if platform == '':
|
try:
|
||||||
sys.stderr.write('Failed to detect platform ' +
|
zf = zipfile.open(package.zipname)
|
||||||
'from package: %r\n' % packagename)
|
f = open(filepath, "wb")
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# extract
|
shutil.copyfileobj(zf, f)
|
||||||
download_prefix = "/data/www/vhosts/builder.blender.org/webroot/download/"
|
os.chmod(filepath, 0o644)
|
||||||
if not branch or branch == 'master':
|
|
||||||
directory = download_prefix
|
|
||||||
elif branch == 'experimental-build':
|
|
||||||
directory = os.path.join(download_prefix, "experimental")
|
|
||||||
else:
|
|
||||||
directory = os.path.join(download_prefix, branch)
|
|
||||||
os.makedirs(directory, exist_ok=True)
|
|
||||||
|
|
||||||
try:
|
zf.close()
|
||||||
filename = os.path.join(directory, packagename)
|
except Exception as ex:
|
||||||
zf = z.open(package)
|
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
|
||||||
f = open(filename, "wb")
|
sys.exit(1)
|
||||||
|
|
||||||
shutil.copyfileobj(zf, f)
|
def remove_older_packages(branch, platform, new_packages):
|
||||||
os.chmod(filename, 0o644)
|
# Remove other files from the same platform and branch
|
||||||
|
directory = get_download_dir(branch)
|
||||||
|
|
||||||
zf.close()
|
for filename in os.listdir(directory):
|
||||||
z.close()
|
package = Package(filename)
|
||||||
except Exception as ex:
|
if package.platform == platform and package.branch == branch:
|
||||||
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
|
is_new_package = False
|
||||||
sys.exit(1)
|
for new_package in new_packages:
|
||||||
|
if package.filename == new_package.filename:
|
||||||
|
is_new_package = True
|
||||||
|
|
||||||
# remove other files from the same platform and branch
|
if not is_new_package:
|
||||||
try:
|
try:
|
||||||
for f in os.listdir(directory):
|
os.remove(os.path.join(directory, filename))
|
||||||
if get_platform(f) == platform and get_branch(f) == branch:
|
except Exception as ex:
|
||||||
if f != packagename:
|
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
|
||||||
os.remove(os.path.join(directory, f))
|
|
||||||
except Exception as ex:
|
|
||||||
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
|
if __name__ == "__main__":
|
||||||
sys.exit(1)
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('filename')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
with open_zipfile(args.filename) as zipfile:
|
||||||
|
packages = get_zipfile_packages(zipfile)
|
||||||
|
branch, platform = get_branch_platform(packages)
|
||||||
|
extract_zipfile_packages(zipfile, packages, branch)
|
||||||
|
|
||||||
|
remove_older_packages(branch, platform, packages)
|
||||||
|
|||||||
Reference in New Issue
Block a user