From d7e4995cfac150011f3eeca7e9c9bc176bb05773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 7 Mar 2017 22:41:05 +0100 Subject: [PATCH] WIP: more work on the docker structure, still not finished with 4_run 1_base: builds a base image, based on Ubuntu 16.10 2_buildpy: builds two images: 2a: an image that can build Python 3.6 2b: an image that contains the built Python 3.6 in /opt/python 3_buildwheels: builds an image to build wheel files, puts them in ../4_run 4_run: the production runtime image, which can't build anything and just runs. --- .gitignore | 3 +- docker/1_base/base.docker | 5 +- docker/2_build/build-wheels.sh | 6 --- docker/2_build/build.docker | 48 ------------------- docker/2_build/build.sh | 33 ------------- docker/2_build/myconfigure | 8 ---- .../Python-3.6.0.tar.xz.sha256 | 0 docker/2_buildpy/build.sh | 39 +++++++++++++++ docker/2_buildpy/buildpy.docker | 23 +++++++++ docker/2_buildpy/includepy.docker | 9 ++++ docker/3_buildwheels/build.docker | 15 ++++++ docker/3_buildwheels/build.sh | 43 +++++++++++++++++ docker/3_run/build.sh | 5 -- docker/{3_run => 4_run}/000-default.conf | 0 docker/{3_run => 4_run}/apache2.conf | 0 docker/4_run/build.sh | 3 ++ docker/{3_run => 4_run}/docker-entrypoint.sh | 0 docker/{3_run => 4_run}/manage.sh | 0 docker/{3_run => 4_run}/run.docker | 10 ++-- docker/build.sh | 19 ++++---- 20 files changed, 150 insertions(+), 119 deletions(-) delete mode 100755 docker/2_build/build-wheels.sh delete mode 100644 docker/2_build/build.docker delete mode 100755 docker/2_build/build.sh delete mode 100755 docker/2_build/myconfigure rename docker/{2_build => 2_buildpy}/Python-3.6.0.tar.xz.sha256 (100%) create mode 100755 docker/2_buildpy/build.sh create mode 100644 docker/2_buildpy/buildpy.docker create mode 100644 docker/2_buildpy/includepy.docker create mode 100644 docker/3_buildwheels/build.docker create mode 100755 docker/3_buildwheels/build.sh delete mode 100755 docker/3_run/build.sh rename docker/{3_run => 4_run}/000-default.conf (100%) rename docker/{3_run => 4_run}/apache2.conf (100%) create mode 100755 docker/4_run/build.sh rename docker/{3_run => 4_run}/docker-entrypoint.sh (100%) rename docker/{3_run => 4_run}/manage.sh (100%) rename docker/{3_run => 4_run}/run.docker (83%) diff --git a/.gitignore b/.gitignore index 40da0fb..ca528d0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ __pycache__ /.eggs/ /dump/ /google_app*.json -/docker/3_run/wheelhouse/ +/docker/2_buildpy/python/ +/docker/4_run/wheelhouse/ diff --git a/docker/1_base/base.docker b/docker/1_base/base.docker index e8bac73..66865f2 100644 --- a/docker/1_base/base.docker +++ b/docker/1_base/base.docker @@ -2,9 +2,8 @@ FROM ubuntu:16.10 MAINTAINER Francesco Siddi RUN apt-get update -RUN apt-get -y dist-upgrade RUN apt-get install -qyy \ -o APT::Install-Recommends=false -o APT::Install-Suggests=false \ - openssl + openssl ca-certificates -RUN mkdir -p /data/git/{pillar,config,venv,wheelhouse,python} +RUN mkdir -p /data/git/pillar /data/config /data/venv /data/wheelhouse diff --git a/docker/2_build/build-wheels.sh b/docker/2_build/build-wheels.sh deleted file mode 100755 index fbbd813..0000000 --- a/docker/2_build/build-wheels.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -cd /data/topdev/blender-cloud -source /data/venv/bin/activate -pip wheel --wheel-dir=/data/wheelhouse -r requirements.txt diff --git a/docker/2_build/build.docker b/docker/2_build/build.docker deleted file mode 100644 index aba8b71..0000000 --- a/docker/2_build/build.docker +++ /dev/null @@ -1,48 +0,0 @@ -FROM pillar_base -MAINTAINER Francesco Siddi - -RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -qy \ - git \ - build-essential \ - checkinstall \ - curl - -# \ -# libffi-dev \ -# libssl-dev \ -# python3.6-dev \ -# python3.6-imaging \ -# zlib1g-dev \ -# libjpeg-dev \ -# libtiff-dev \ -# python3.6-crypto \ -# python3.6-openssl - -RUN apt-get build-dep -y python3.5 - -ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256 - -RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz -RUN sha256sum -c Python-3.6.0.tar.xz.sha256 -RUN tar xvf Python-3.6.0.tar.xz - -ADD myconfigure /Python-3.6.0/myconfigure -# RUN cd Python-3.6.0/ && ./myconfigure -# RUN cd Python-3.6.0/ && make -j8 install - -ENV PYTHONTARGET=/data/python -ENV WHEELHOUSE=/data/wheelhouse -ENV PIP_WHEEL_DIR=/data/wheelhouse -ENV PIP_FIND_LINKS=/data/wheelhouse - -VOLUME /data/wheelhouse -VOLUME /data/python - -# RUN /data/python/bin/python3.6 -m venv /data/venv -# RUN . /data/venv/bin/activate && pip install -U pip -# RUN . /data/venv/bin/activate && pip install wheel - -ADD build-wheels.sh /build-wheels.sh -ENTRYPOINT ["bash", "build-wheels.sh"] diff --git a/docker/2_build/build.sh b/docker/2_build/build.sh deleted file mode 100755 index 362b485..0000000 --- a/docker/2_build/build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# macOS does not support readlink -f, so we use greadlink instead -if [ $(uname) == 'Darwin' ]; then - command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } - readlink='greadlink' -else - readlink='readlink' -fi - -TOPDEVDIR="$($readlink -f ../../..)" -echo "Top-level development dir is $TOPDEVDIR" - -PYTHON=$($readlink -f ../3_run/python) -WHEELHOUSE="$($readlink -f ../3_run/wheelhouse)" -if [ -z "$WHEELHOUSE" -o -z "$PYTHON" ]; then - echo "Error, ../3_run might not exist." >&2 - exit 2 -fi - -mkdir -p "$WHEELHOUSE" "$PYTHON" -echo "Wheelhouse is $WHEELHOUSE" -echo "Python will be built to $PYTHON" - -docker build -t pillar_build -f build.docker . -#docker run --rm \ -# -v "$WHEELHOUSE:/data/wheelhouse" \ -# -v "$TOPDEVDIR:/data/topdev" \ -# -v "$PYTHON:/data/python" \ -# pillar_build - -# RUN cd Python-3.6.0/ && ./myconfigure -# RUN cd Python-3.6.0/ && make -j8 install diff --git a/docker/2_build/myconfigure b/docker/2_build/myconfigure deleted file mode 100755 index 862dd2f..0000000 --- a/docker/2_build/myconfigure +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -./configure \ - --prefix=$PYTHONTARGET \ - --enable-ipv6 \ - --enable-optimizations \ - --enable-shared \ - --with-ensurepip=upgrade diff --git a/docker/2_build/Python-3.6.0.tar.xz.sha256 b/docker/2_buildpy/Python-3.6.0.tar.xz.sha256 similarity index 100% rename from docker/2_build/Python-3.6.0.tar.xz.sha256 rename to docker/2_buildpy/Python-3.6.0.tar.xz.sha256 diff --git a/docker/2_buildpy/build.sh b/docker/2_buildpy/build.sh new file mode 100755 index 0000000..665385d --- /dev/null +++ b/docker/2_buildpy/build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -e + +# macOS does not support readlink -f, so we use greadlink instead +if [ $(uname) == 'Darwin' ]; then + command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } + readlink='greadlink' +else + readlink='readlink' +fi + +PYTHONTARGET=$($readlink -f ./python) + +mkdir -p "$PYTHONTARGET" +echo "Python will be built to $PYTHONTARGET" + +docker build -t pillar_build -f buildpy.docker . + +# Use the docker image to build Python 3.6. +GID=$(id --group) +docker run --rm -i \ + -v "$PYTHONTARGET:/opt/python" \ + pillar_build < + +RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -qy \ + build-essential \ + checkinstall \ + curl + +RUN apt-get build-dep -y python3.5 + +ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256 + +RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz +RUN sha256sum -c Python-3.6.0.tar.xz.sha256 +RUN tar xf Python-3.6.0.tar.xz +RUN rm Python-3.6.0.tar.xz + +# To be able to install Python outside the docker. +VOLUME /opt/python + +ENV PYTHONSOURCE=/Python-3.6.0 diff --git a/docker/2_buildpy/includepy.docker b/docker/2_buildpy/includepy.docker new file mode 100644 index 0000000..47b7ee2 --- /dev/null +++ b/docker/2_buildpy/includepy.docker @@ -0,0 +1,9 @@ +FROM pillar_base +MAINTAINER Sybren A. Stüvel + +ADD python /opt/python +RUN echo /opt/python/lib > /etc/ld.so.conf.d/python.conf +RUN ldconfig + +RUN echo Python is installed in /opt/python/ > README.python +ENV PATH=/opt/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/docker/3_buildwheels/build.docker b/docker/3_buildwheels/build.docker new file mode 100644 index 0000000..37dded2 --- /dev/null +++ b/docker/3_buildwheels/build.docker @@ -0,0 +1,15 @@ +FROM pillar_py:3.6 +MAINTAINER Sybren A. Stüvel + +RUN apt-get install -qy \ + git \ + build-essential \ + checkinstall \ + libffi-dev \ + libssl-dev + +ENV WHEELHOUSE=/data/wheelhouse +ENV PIP_WHEEL_DIR=/data/wheelhouse +ENV PIP_FIND_LINKS=/data/wheelhouse + +VOLUME /data/wheelhouse diff --git a/docker/3_buildwheels/build.sh b/docker/3_buildwheels/build.sh new file mode 100755 index 0000000..4d5c5cd --- /dev/null +++ b/docker/3_buildwheels/build.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# macOS does not support readlink -f, so we use greadlink instead +if [ $(uname) == 'Darwin' ]; then + command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } + readlink='greadlink' +else + readlink='readlink' +fi + +TOPDEVDIR="$($readlink -f ../../..)" +echo "Top-level development dir is $TOPDEVDIR" + +WHEELHOUSE="$($readlink -f ../4_run/wheelhouse)" +if [ -z "$WHEELHOUSE" ]; then + echo "Error, ../4_run might not exist." >&2 + exit 2 +fi + +echo "Wheelhouse is $WHEELHOUSE" +mkdir -p "$WHEELHOUSE" + +#docker build -t pillar_wheelbuilder -f build.docker . + +GID=$(id --group) +docker run --rm -i \ + -v "$WHEELHOUSE:/data/wheelhouse" \ + -v "$TOPDEVDIR:/data/topdev" \ + pillar_wheelbuilder < /data/wheelhouse/requirements.txt +EOT + +# Remove our own projects, they shouldn't be installed as wheel (for now). +rm -f $WHEELHOUSE/{attract,flamenco,pillar,pillarsdk}*.whl diff --git a/docker/3_run/build.sh b/docker/3_run/build.sh deleted file mode 100755 index 5853b58..0000000 --- a/docker/3_run/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -cp ../../requirements.txt .; -docker build -t armadillica/blender_cloud -f run.docker .; -rm requirements.txt; diff --git a/docker/3_run/000-default.conf b/docker/4_run/000-default.conf similarity index 100% rename from docker/3_run/000-default.conf rename to docker/4_run/000-default.conf diff --git a/docker/3_run/apache2.conf b/docker/4_run/apache2.conf similarity index 100% rename from docker/3_run/apache2.conf rename to docker/4_run/apache2.conf diff --git a/docker/4_run/build.sh b/docker/4_run/build.sh new file mode 100755 index 0000000..f69c074 --- /dev/null +++ b/docker/4_run/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t armadillica/blender_cloud -f run.docker . diff --git a/docker/3_run/docker-entrypoint.sh b/docker/4_run/docker-entrypoint.sh similarity index 100% rename from docker/3_run/docker-entrypoint.sh rename to docker/4_run/docker-entrypoint.sh diff --git a/docker/3_run/manage.sh b/docker/4_run/manage.sh similarity index 100% rename from docker/3_run/manage.sh rename to docker/4_run/manage.sh diff --git a/docker/3_run/run.docker b/docker/4_run/run.docker similarity index 83% rename from docker/3_run/run.docker rename to docker/4_run/run.docker index af3d4e0..6c7c572 100755 --- a/docker/3_run/run.docker +++ b/docker/4_run/run.docker @@ -1,4 +1,4 @@ -FROM pillar_base +FROM pillar_py:3.6 RUN apt-get update && apt-get install -qyy \ -o APT::Install-Recommends=true -o APT::Install-Suggests=false \ @@ -22,16 +22,14 @@ ENV APACHE_LOCK_DIR /var/lock/apache2 RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR -ADD requirements.txt /requirements.txt ADD wheelhouse /data/wheelhouse - -RUN . /data/venv/bin/activate \ -&& pip install --no-index --find-links=/data/wheelhouse -r requirements.txt \ -&& rm /requirements.txt +RUN pip3 install --no-index --find-links=/data/wheelhouse -r /data/wheelhouse/requirements.txt VOLUME /data/git/blender-cloud VOLUME /data/git/pillar VOLUME /data/git/pillar-python-sdk +VOLUME /data/git/attract +VOLUME /data/git/flamenco VOLUME /data/config VOLUME /data/storage diff --git a/docker/build.sh b/docker/build.sh index 5594bae..bd7a41d 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash -set -x; -set -e; +set -xe DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $DIR; -cd 1_base/; -bash build.sh; +cd $DIR/1_base +bash build.sh -cd ../2_build/; -bash build.sh; +cd $DIR/2_buildpy +bash build.sh -cd ../3_run/; -bash build.sh; +cd $DIR/3_buildwheels +bash build.sh + +cd $DIR/4_run +bash build.sh