WIP: more work on the docker structure, still not finished with 4_run

1_base: builds a base image, based on Ubuntu 16.10
2_buildpy: builds two images:
	2a: an image that can build Python 3.6
	2b: an image that contains the built Python 3.6 in /opt/python
3_buildwheels: builds an image to build wheel files, puts them in ../4_run
4_run: the production runtime image, which can't build anything and just runs.

.gitignore

@@ -12,4 +12,5 @@ __pycache__
 /.eggs/
 /dump/
 /google_app*.json
-/docker/3_run/wheelhouse/
+/docker/2_buildpy/python/
+/docker/4_run/wheelhouse/

docker/1_base/base.docker

@@ -2,9 +2,8 @@ FROM ubuntu:16.10
 MAINTAINER Francesco Siddi <francesco@blender.org>
 
 RUN apt-get update
-RUN apt-get -y dist-upgrade
 RUN apt-get install -qyy \
     -o APT::Install-Recommends=false -o APT::Install-Suggests=false \
-    openssl
+    openssl ca-certificates
 
-RUN mkdir -p /data/git/{pillar,config,venv,wheelhouse,python}
+RUN mkdir -p /data/git/pillar /data/config /data/venv /data/wheelhouse

docker/2_build/build-wheels.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-cd /data/topdev/blender-cloud
-source /data/venv/bin/activate
-pip wheel --wheel-dir=/data/wheelhouse -r requirements.txt

docker/2_build/build.docker

@@ -1,48 +0,0 @@
-FROM pillar_base
-MAINTAINER Francesco Siddi <francesco@blender.org>
-
-RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -qy \
-        git \
-        build-essential \
-        checkinstall \
-        curl
-
-#  \
-# libffi-dev \
-# libssl-dev \
-# python3.6-dev \
-# python3.6-imaging \
-# zlib1g-dev \
-# libjpeg-dev \
-# libtiff-dev \
-# python3.6-crypto \
-# python3.6-openssl
-
-RUN apt-get build-dep -y python3.5
-
-ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256
-
-RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz
-RUN sha256sum -c Python-3.6.0.tar.xz.sha256
-RUN tar xvf Python-3.6.0.tar.xz
-
-ADD myconfigure /Python-3.6.0/myconfigure
-# RUN cd Python-3.6.0/ && ./myconfigure
-# RUN cd Python-3.6.0/ && make -j8 install
-
-ENV PYTHONTARGET=/data/python
-ENV WHEELHOUSE=/data/wheelhouse
-ENV PIP_WHEEL_DIR=/data/wheelhouse
-ENV PIP_FIND_LINKS=/data/wheelhouse
-
-VOLUME /data/wheelhouse
-VOLUME /data/python
-
-# RUN /data/python/bin/python3.6 -m venv /data/venv
-# RUN . /data/venv/bin/activate && pip install -U pip
-# RUN . /data/venv/bin/activate && pip install wheel
-
-ADD build-wheels.sh /build-wheels.sh
-ENTRYPOINT ["bash", "build-wheels.sh"]

docker/2_build/build.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# macOS does not support readlink -f, so we use greadlink instead
-if [ $(uname) == 'Darwin' ]; then
-    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
-    readlink='greadlink'
-else
-    readlink='readlink'
-fi
-
-TOPDEVDIR="$($readlink -f ../../..)"
-echo "Top-level development dir is $TOPDEVDIR"
-
-PYTHON=$($readlink -f ../3_run/python)
-WHEELHOUSE="$($readlink -f ../3_run/wheelhouse)"
-if [ -z "$WHEELHOUSE" -o -z "$PYTHON" ]; then
-    echo "Error, ../3_run might not exist." >&2
-    exit 2
-fi
-
-mkdir -p "$WHEELHOUSE" "$PYTHON"
-echo "Wheelhouse is $WHEELHOUSE"
-echo "Python will be built to $PYTHON"
-
-docker build -t pillar_build -f build.docker .
-#docker run --rm \
-#    -v "$WHEELHOUSE:/data/wheelhouse" \
-#    -v "$TOPDEVDIR:/data/topdev" \
-#    -v "$PYTHON:/data/python" \
-#    pillar_build
-
-# RUN cd Python-3.6.0/ && ./myconfigure
-# RUN cd Python-3.6.0/ && make -j8 install

docker/2_build/myconfigure

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-./configure \
-    --prefix=$PYTHONTARGET \
-    --enable-ipv6 \
-    --enable-optimizations \
-    --enable-shared \
-    --with-ensurepip=upgrade

docker/2_buildpy/build.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -e
+
+# macOS does not support readlink -f, so we use greadlink instead
+if [ $(uname) == 'Darwin' ]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
+
+PYTHONTARGET=$($readlink -f ./python)
+
+mkdir -p "$PYTHONTARGET"
+echo "Python will be built to $PYTHONTARGET"
+
+docker build -t pillar_build -f buildpy.docker .
+
+# Use the docker image to build Python 3.6.
+GID=$(id --group)
+docker run --rm -i \
+    -v "$PYTHONTARGET:/opt/python" \
+    pillar_build <<EOT
+set -e
+cd \$PYTHONSOURCE
+./configure \
+    --prefix=/opt/python \
+    --enable-ipv6 \
+    --enable-shared \
+    --with-ensurepip=upgrade
+make -j8 install
+chown -R $UID:$GID /opt/python/*
+EOT
+
+# Create another docker image which contains the actual Python.
+# This one will serve as base for the Wheel builder and the
+# production image.
+docker build -t pillar_py:3.6 -f includepy.docker .

docker/2_buildpy/buildpy.docker

@@ -0,0 +1,23 @@
+FROM pillar_base
+MAINTAINER Sybren A. Stüvel <sybren@blender.studio>
+
+RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -qy \
+        build-essential \
+        checkinstall \
+        curl
+
+RUN apt-get build-dep -y python3.5
+
+ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256
+
+RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz
+RUN sha256sum -c Python-3.6.0.tar.xz.sha256
+RUN tar xf Python-3.6.0.tar.xz
+RUN rm Python-3.6.0.tar.xz
+
+# To be able to install Python outside the docker.
+VOLUME /opt/python
+
+ENV PYTHONSOURCE=/Python-3.6.0

docker/2_buildpy/includepy.docker

@@ -0,0 +1,9 @@
+FROM pillar_base
+MAINTAINER Sybren A. Stüvel <sybren@blender.studio>
+
+ADD python /opt/python
+RUN echo /opt/python/lib > /etc/ld.so.conf.d/python.conf
+RUN ldconfig
+
+RUN echo Python is installed in /opt/python/ > README.python
+ENV PATH=/opt/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

docker/3_buildwheels/build.docker

@@ -0,0 +1,15 @@
+FROM pillar_py:3.6
+MAINTAINER Sybren A. Stüvel <sybren@blender.studio>
+
+RUN apt-get install -qy \
+    git \
+    build-essential \
+    checkinstall \
+    libffi-dev \
+    libssl-dev
+
+ENV WHEELHOUSE=/data/wheelhouse
+ENV PIP_WHEEL_DIR=/data/wheelhouse
+ENV PIP_FIND_LINKS=/data/wheelhouse
+
+VOLUME /data/wheelhouse

docker/3_buildwheels/build.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# macOS does not support readlink -f, so we use greadlink instead
+if [ $(uname) == 'Darwin' ]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
+
+TOPDEVDIR="$($readlink -f ../../..)"
+echo "Top-level development dir is $TOPDEVDIR"
+
+WHEELHOUSE="$($readlink -f ../4_run/wheelhouse)"
+if [ -z "$WHEELHOUSE" ]; then
+    echo "Error, ../4_run might not exist." >&2
+    exit 2
+fi
+
+echo "Wheelhouse is $WHEELHOUSE"
+mkdir -p "$WHEELHOUSE"
+
+#docker build -t pillar_wheelbuilder -f build.docker .
+
+GID=$(id --group)
+docker run --rm -i \
+    -v "$WHEELHOUSE:/data/wheelhouse" \
+    -v "$TOPDEVDIR:/data/topdev" \
+    pillar_wheelbuilder <<EOT
+set -e
+# Build wheels for all dependencies.
+cd /data/topdev/blender-cloud
+# pip3 install wheel
+# pip3 wheel --wheel-dir=/data/wheelhouse -r requirements.txt
+# chown -R $UID:$GID /data/wheelhouse
+
+# Install the dependencies so that we can get a full freeze.
+pip3 install --no-index --find-links=/data/wheelhouse -r requirements.txt
+pip3 freeze | grep -v '^-[ef] ' > /data/wheelhouse/requirements.txt
+EOT
+
+# Remove our own projects, they shouldn't be installed as wheel (for now).
+rm -f $WHEELHOUSE/{attract,flamenco,pillar,pillarsdk}*.whl

docker/3_run/build.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-cp ../../requirements.txt .;
-docker build -t armadillica/blender_cloud -f run.docker .;
-rm requirements.txt;

docker/4_run/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker build -t armadillica/blender_cloud -f run.docker .

docker/4_run/run.docker

@@ -1,4 +1,4 @@
-FROM pillar_base
+FROM pillar_py:3.6
 
 RUN apt-get update && apt-get install -qyy \
 -o APT::Install-Recommends=true -o APT::Install-Suggests=false \
@@ -22,16 +22,14 @@ ENV APACHE_LOCK_DIR /var/lock/apache2
 
 RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR
 
-ADD requirements.txt /requirements.txt
 ADD wheelhouse /data/wheelhouse
-
+RUN pip3 install --no-index --find-links=/data/wheelhouse -r /data/wheelhouse/requirements.txt
-RUN . /data/venv/bin/activate \
-&& pip install --no-index --find-links=/data/wheelhouse -r requirements.txt \
-&& rm /requirements.txt
 
 VOLUME /data/git/blender-cloud
 VOLUME /data/git/pillar
 VOLUME /data/git/pillar-python-sdk
+VOLUME /data/git/attract
+VOLUME /data/git/flamenco
 VOLUME /data/config
 VOLUME /data/storage
 

docker/build.sh

@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 
-set -x;
+set -xe
-set -e;
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd $DIR;
 
-cd 1_base/;
+cd $DIR/1_base
-bash build.sh;
+bash build.sh
 
-cd ../2_build/;
+cd $DIR/2_buildpy
-bash build.sh;
+bash build.sh
 
-cd ../3_run/;
+cd $DIR/3_buildwheels
-bash build.sh;
+bash build.sh
+
+cd $DIR/4_run
+bash build.sh