From af14910fa9edde63a98e8ebbe7b14175119ad68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 7 Mar 2017 16:51:51 +0100 Subject: [PATCH] WIP breaking stuff: updating docker image build process for Python 3.6 This requires a new way to pass requirements.txt files to Docker (since they now link to each other), as well as building Python ourselves (since even Ubuntu 16.10 doesn't have a decent Python 3.6). This is just a WIP commit, will be fixed soon(ish). --- docker/1_base/base.docker | 18 +++----- docker/1_base/build.sh | 0 docker/2_build/Python-3.6.0.tar.xz.sha256 | 1 + docker/2_build/build-wheels.sh | 5 ++- docker/2_build/build.docker | 50 ++++++++++++++++------- docker/2_build/build.sh | 36 ++++++++++++---- docker/2_build/myconfigure | 8 ++++ docker/3_run/run.docker | 2 + 8 files changed, 86 insertions(+), 34 deletions(-) mode change 100755 => 100644 docker/1_base/base.docker mode change 100644 => 100755 docker/1_base/build.sh create mode 100644 docker/2_build/Python-3.6.0.tar.xz.sha256 mode change 100644 => 100755 docker/2_build/build-wheels.sh mode change 100755 => 100644 docker/2_build/build.docker create mode 100755 docker/2_build/myconfigure diff --git a/docker/1_base/base.docker b/docker/1_base/base.docker old mode 100755 new mode 100644 index 04805ab..e8bac73 --- a/docker/1_base/base.docker +++ b/docker/1_base/base.docker @@ -1,16 +1,10 @@ -FROM ubuntu:16.04 +FROM ubuntu:16.10 MAINTAINER Francesco Siddi -RUN apt-get update && apt-get install -qyy \ +RUN apt-get update +RUN apt-get -y dist-upgrade +RUN apt-get install -qyy \ -o APT::Install-Recommends=false -o APT::Install-Suggests=false \ - python-pip libffi6 openssl ffmpeg rsyslog logrotate + openssl -RUN mkdir -p /data/git/pillar \ -&& mkdir -p /data/storage \ -&& mkdir -p /data/config \ -&& mkdir -p /data/venv \ -&& mkdir -p /data/wheelhouse - -RUN pip install virtualenv -RUN virtualenv /data/venv -RUN . /data/venv/bin/activate && pip install -U pip && pip install wheel +RUN mkdir -p /data/git/{pillar,config,venv,wheelhouse,python} diff --git a/docker/1_base/build.sh b/docker/1_base/build.sh old mode 100644 new mode 100755 diff --git a/docker/2_build/Python-3.6.0.tar.xz.sha256 b/docker/2_build/Python-3.6.0.tar.xz.sha256 new file mode 100644 index 0000000..c102266 --- /dev/null +++ b/docker/2_build/Python-3.6.0.tar.xz.sha256 @@ -0,0 +1 @@ +b0c5f904f685e32d9232f7bdcbece9819a892929063b6e385414ad2dd6a23622 Python-3.6.0.tar.xz diff --git a/docker/2_build/build-wheels.sh b/docker/2_build/build-wheels.sh old mode 100644 new mode 100755 index 56cc2bc..fbbd813 --- a/docker/2_build/build-wheels.sh +++ b/docker/2_build/build-wheels.sh @@ -1,3 +1,6 @@ #!/usr/bin/env bash -. /data/venv/bin/activate && pip wheel --wheel-dir=/data/wheelhouse -r /requirements.txt +set -e +cd /data/topdev/blender-cloud +source /data/venv/bin/activate +pip wheel --wheel-dir=/data/wheelhouse -r requirements.txt diff --git a/docker/2_build/build.docker b/docker/2_build/build.docker old mode 100755 new mode 100644 index 0a5668b..aba8b71 --- a/docker/2_build/build.docker +++ b/docker/2_build/build.docker @@ -1,26 +1,48 @@ FROM pillar_base MAINTAINER Francesco Siddi -RUN apt-get update && apt-get install -qy \ -git \ -gcc \ -libffi-dev \ -libssl-dev \ -pypy-dev \ -python-dev \ -python-imaging \ -zlib1g-dev \ -libjpeg-dev \ -libtiff-dev \ -python-crypto \ -python-openssl +RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -qy \ + git \ + build-essential \ + checkinstall \ + curl +# \ +# libffi-dev \ +# libssl-dev \ +# python3.6-dev \ +# python3.6-imaging \ +# zlib1g-dev \ +# libjpeg-dev \ +# libtiff-dev \ +# python3.6-crypto \ +# python3.6-openssl + +RUN apt-get build-dep -y python3.5 + +ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256 + +RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz +RUN sha256sum -c Python-3.6.0.tar.xz.sha256 +RUN tar xvf Python-3.6.0.tar.xz + +ADD myconfigure /Python-3.6.0/myconfigure +# RUN cd Python-3.6.0/ && ./myconfigure +# RUN cd Python-3.6.0/ && make -j8 install + +ENV PYTHONTARGET=/data/python ENV WHEELHOUSE=/data/wheelhouse ENV PIP_WHEEL_DIR=/data/wheelhouse ENV PIP_FIND_LINKS=/data/wheelhouse VOLUME /data/wheelhouse +VOLUME /data/python + +# RUN /data/python/bin/python3.6 -m venv /data/venv +# RUN . /data/venv/bin/activate && pip install -U pip +# RUN . /data/venv/bin/activate && pip install wheel -ADD requirements.txt /requirements.txt ADD build-wheels.sh /build-wheels.sh ENTRYPOINT ["bash", "build-wheels.sh"] diff --git a/docker/2_build/build.sh b/docker/2_build/build.sh index 533b149..362b485 100755 --- a/docker/2_build/build.sh +++ b/docker/2_build/build.sh @@ -1,11 +1,33 @@ #!/usr/bin/env bash -mkdir -p ../3_run/wheelhouse; -cp ../../requirements.txt .; +# macOS does not support readlink -f, so we use greadlink instead +if [ $(uname) == 'Darwin' ]; then + command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } + readlink='greadlink' +else + readlink='readlink' +fi -docker build -t pillar_build -f build.docker .; -docker run --rm \ - -v "$(pwd)"/../3_run/wheelhouse:/data/wheelhouse \ - pillar_build; +TOPDEVDIR="$($readlink -f ../../..)" +echo "Top-level development dir is $TOPDEVDIR" -rm requirements.txt; +PYTHON=$($readlink -f ../3_run/python) +WHEELHOUSE="$($readlink -f ../3_run/wheelhouse)" +if [ -z "$WHEELHOUSE" -o -z "$PYTHON" ]; then + echo "Error, ../3_run might not exist." >&2 + exit 2 +fi + +mkdir -p "$WHEELHOUSE" "$PYTHON" +echo "Wheelhouse is $WHEELHOUSE" +echo "Python will be built to $PYTHON" + +docker build -t pillar_build -f build.docker . +#docker run --rm \ +# -v "$WHEELHOUSE:/data/wheelhouse" \ +# -v "$TOPDEVDIR:/data/topdev" \ +# -v "$PYTHON:/data/python" \ +# pillar_build + +# RUN cd Python-3.6.0/ && ./myconfigure +# RUN cd Python-3.6.0/ && make -j8 install diff --git a/docker/2_build/myconfigure b/docker/2_build/myconfigure new file mode 100755 index 0000000..862dd2f --- /dev/null +++ b/docker/2_build/myconfigure @@ -0,0 +1,8 @@ +#!/bin/bash + +./configure \ + --prefix=$PYTHONTARGET \ + --enable-ipv6 \ + --enable-optimizations \ + --enable-shared \ + --with-ensurepip=upgrade diff --git a/docker/3_run/run.docker b/docker/3_run/run.docker index 8c41d54..af3d4e0 100755 --- a/docker/3_run/run.docker +++ b/docker/3_run/run.docker @@ -8,6 +8,8 @@ libapache2-mod-wsgi \ libapache2-mod-xsendfile \ libjpeg8 \ libtiff5 \ +ffmpeg \ +rsyslog logrotate \ nano vim curl \ && rm -rf /var/lib/apt/lists/*