From cbdf1c1321099745369dae2c10b1a22c94d9df7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Thu, 31 Mar 2016 17:33:48 +0200 Subject: [PATCH] Download & bundle dependencies as wheel files. build-dependency-wheels.sh has been removed, as it has been superseded by the 'python setup.py wheels' command. --- build-dependency-wheels.sh | 44 ------------- setup.py | 129 +++++++++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 48 deletions(-) delete mode 100755 build-dependency-wheels.sh diff --git a/build-dependency-wheels.sh b/build-dependency-wheels.sh deleted file mode 100755 index fa38503..0000000 --- a/build-dependency-wheels.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -MYDIR=$(dirname $(readlink -f $0)) -WHEELS=$MYDIR/blender_cloud/wheels -cd $MYDIR - -PILLAR_SDK_DIR=$MYDIR/../pillar-python-sdk -CACHECONTROL_DIR=$MYDIR/../cachecontrol - -# Build the Pillar Python SDK wheel from ../pillar-python-sdk -if [ ! -e $WHEELS/lockfile*.whl ]; then - echo "Building pillar_sdk wheel" - if [ ! -e $PILLAR_SDK_DIR ]; then - cd $(dirname $PILLAR_SDK_DIR) - git clone https://github.com/armadillica/pillar-python-sdk.git $PILLAR_SDK_DIR - fi - - cd $PILLAR_SDK_DIR - python setup.py bdist_wheel - cp $(ls dist/*.whl -rt | tail -n 1) $WHEELS -fi - -# Download lockfile wheel -if [ ! -e $WHEELS/lockfile*.whl ]; then - echo "Downloading lockfile" - pip download --dest $WHEELS $(grep -i lockfile $MYDIR/requirements.txt) -fi - -# Build CacheControl wheel -if [ ! -e $WHEELS/CacheControl*.whl ]; then - echo "Building CacheControl wheel" - if [ ! -e $CACHECONTROL_DIR ]; then - cd $(dirname $CACHECONTROL_DIR) - git clone https://github.com/ionrock/cachecontrol.git $CACHECONTROL_DIR - cd $CACHECONTROL_DIR - git checkout v0.11.6 # TODO: get from requirements.txt - fi - - cd $CACHECONTROL_DIR - rm -f dist/*.whl - python setup.py bdist_wheel - cp $(ls dist/*.whl -rt | tail -n 1) $WHEELS -fi - diff --git a/setup.py b/setup.py index 4a27830..65aaa50 100755 --- a/setup.py +++ b/setup.py @@ -1,11 +1,128 @@ #!/usr/bin/env python +import sys +import shutil +import subprocess +import re +import pathlib +from glob import glob +from distutils import log +from distutils.core import Command from distutils.command.bdist import bdist from distutils.command.install import install from distutils.command.install_egg_info import install_egg_info from setuptools import setup, find_packages -from glob import glob +requirement_re = re.compile('[><=]+') + + +class BuildWheels(Command): + """Builds or downloads the dependencies as wheel files.""" + + description = "builds/downloads the dependencies as wheel files" + user_options = [ + ('wheels-path=', None, "wheel file installation path"), + ('deps-path=', None, "path in which dependencies are built"), + ('pillar-sdk-path=', None, "subdir of deps-path containing the Pillar Python SDK"), + ('cachecontrol-path=', None, "subdir of deps-path containing CacheControl"), + ] + + def initialize_options(self): + self.wheels_path = None # path that will contain the installed wheels. + self.deps_path = None # path in which dependencies are built. + self.pillar_sdk_path = None # subdir of deps_path containing the Pillar Python SDK + self.cachecontrol_path = None # subdir of deps_path containing CacheControl + + def finalize_options(self): + self.my_path = pathlib.Path(__file__).resolve().parent + package_path = self.my_path / self.distribution.get_name() + + def set_default(var, default): + if var is None: + return default + return pathlib.Path(var) # convert CLI-arguments (string) to Paths. + + self.wheels_path = set_default(self.wheels_path, package_path / 'wheels') + self.deps_path = set_default(self.deps_path, self.my_path / 'build/deps') + self.pillar_sdk_path = set_default(self.pillar_sdk_path, + self.deps_path / 'pillar-python-sdk') + self.cachecontrol_path = set_default(self.cachecontrol_path, + self.deps_path / 'cachecontrol') + + def run(self): + log.info('Storing wheels in %s', self.wheels_path) + + # Parse the requirements.txt file + requirements = {} + with open(str(self.my_path / 'requirements.txt')) as reqfile: + for line in reqfile.readlines(): + line = line.strip() + + if not line or line.startswith('#'): + # comments are lines that start with # only + continue + + line_req = requirement_re.split(line) + package = line_req[0] + version = line_req[-1] + requirements[package] = (line, version) + # log.info(' - %s = %s / %s', package, line, line_req[-1]) + + self.wheels_path.mkdir(parents=True, exist_ok=True) + + # Download lockfile, as there is a suitable wheel on pypi. + if not list(self.wheels_path.glob('lockfile*.whl')): + log.info('Downloading lockfile wheel') + subprocess.check_call([ + 'pip', 'download', '--dest', str(self.wheels_path), requirements['lockfile'][0] + ]) + + # Build Pillar Python SDK. + if not list(self.wheels_path.glob('pillar-python-sdk*.whl')): + log.info('Building Pillar Python SDK in %s', self.pillar_sdk_path) + self.git_clone(self.pillar_sdk_path, + 'https://github.com/armadillica/pillar-python-sdk.git') + self.build_copy_wheel(self.pillar_sdk_path) + + # Build CacheControl. + if not list(self.wheels_path.glob('CacheControl*.whl')): + log.info('Building CacheControl in %s', self.cachecontrol_path) + self.git_clone(self.cachecontrol_path, + 'https://github.com/ionrock/cachecontrol.git', + 'v%s' % requirements['CacheControl'][1]) + self.build_copy_wheel(self.cachecontrol_path) + + # Ensure that the wheels are added to the data files. + self.distribution.data_files.append( + ('blender_cloud/wheels', (str(p) for p in self.wheels_path.glob('*.whl'))) + ) + + def git_clone(self, workdir: pathlib.Path, git_url: str, checkout: str = None): + if workdir.exists(): + # Directory exists, expect it to be set up correctly. + return + + workdir.mkdir(parents=True) + + subprocess.check_call(['git', 'clone', git_url, str(workdir)], + cwd=str(workdir.parent)) + + if checkout: + subprocess.check_call(['git', 'checkout', checkout], + cwd=str(workdir)) + + def build_copy_wheel(self, package_path: pathlib.Path): + # Make sure no wheels exist yet, so that we know which one to copy later. + to_remove = list((package_path / 'dist').glob('*.whl')) + for fname in to_remove: + fname.unlink() + + subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel'], + cwd=str(package_path)) + + wheel = next((package_path / 'dist').glob('*.whl')) + log.info('copying %s to %s', wheel, self.wheels_path) + shutil.copy(str(wheel), str(self.wheels_path)) class BlenderAddonBdist(bdist): """Ensures that 'python setup.py bdist' creates a zip file.""" @@ -15,6 +132,10 @@ class BlenderAddonBdist(bdist): self.formats = ['zip'] self.plat_name = 'addon' # use this instead of 'linux-x86_64' or similar. + def run(self): + self.run_command('wheels') + super().run() + class BlenderAddonInstall(install): """Ensures the module is placed at the root of the zip file.""" @@ -39,15 +160,15 @@ class AvoidEggInfo(install_egg_info): setup( cmdclass={'bdist': BlenderAddonBdist, 'install': BlenderAddonInstall, - 'install_egg_info': AvoidEggInfo}, + 'install_egg_info': AvoidEggInfo, + 'wheels': BuildWheels}, name='blender_cloud', description='The Blender Cloud addon allows browsing the Blender Cloud from Blender.', version='1.0.0', author='Sybren A. Stüvel', author_email='sybren@stuvel.eu', packages=find_packages('.'), - data_files=[('blender_cloud', ['README.md']), - ('blender_cloud/wheels', glob('blender_cloud/wheels/*.whl'))], + data_files=[('blender_cloud', ['README.md'])], scripts=[], url='https://developer.blender.org/diffusion/BCA/', license='GNU General Public License v2 or later (GPLv2+)',