WIP breaking stuff: updating docker image build process for Python 3.6

This requires a new way to pass requirements.txt files to Docker (since
they now link to each other), as well as building Python ourselves (since
even Ubuntu 16.10 doesn't have a decent Python 3.6).

This is just a WIP commit, will be fixed soon(ish).

docker/1_base/base.docker

@@ -1,16 +1,10 @@
-FROM ubuntu:16.04
+FROM ubuntu:16.10
 MAINTAINER Francesco Siddi <francesco@blender.org>
 
-RUN apt-get update && apt-get install -qyy \
+RUN apt-get update
+RUN apt-get -y dist-upgrade
+RUN apt-get install -qyy \
     -o APT::Install-Recommends=false -o APT::Install-Suggests=false \
-    python-pip libffi6 openssl ffmpeg rsyslog logrotate
+    openssl
 
-RUN mkdir -p /data/git/pillar \
-&& mkdir -p /data/storage \
-&& mkdir -p /data/config \
-&& mkdir -p /data/venv \
-&& mkdir -p /data/wheelhouse
-
-RUN pip install virtualenv
-RUN virtualenv /data/venv
-RUN . /data/venv/bin/activate && pip install -U pip && pip install wheel
+RUN mkdir -p /data/git/{pillar,config,venv,wheelhouse,python}

docker/2_build/Python-3.6.0.tar.xz.sha256

@@ -0,0 +1 @@
+b0c5f904f685e32d9232f7bdcbece9819a892929063b6e385414ad2dd6a23622  Python-3.6.0.tar.xz

docker/2_build/build-wheels.sh

@@ -1,3 +1,6 @@
 #!/usr/bin/env bash
 
-. /data/venv/bin/activate && pip wheel --wheel-dir=/data/wheelhouse -r /requirements.txt
+set -e
+cd /data/topdev/blender-cloud
+source /data/venv/bin/activate
+pip wheel --wheel-dir=/data/wheelhouse -r requirements.txt

docker/2_build/build.docker

@@ -1,26 +1,48 @@
 FROM pillar_base
 MAINTAINER Francesco Siddi <francesco@blender.org>
 
-RUN apt-get update && apt-get install -qy \
-git \
-gcc \
-libffi-dev \
-libssl-dev \
-pypy-dev \
-python-dev \
-python-imaging \
-zlib1g-dev \
-libjpeg-dev \
-libtiff-dev \
-python-crypto \
-python-openssl
+RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -qy \
+        git \
+        build-essential \
+        checkinstall \
+        curl
 
+#  \
+# libffi-dev \
+# libssl-dev \
+# python3.6-dev \
+# python3.6-imaging \
+# zlib1g-dev \
+# libjpeg-dev \
+# libtiff-dev \
+# python3.6-crypto \
+# python3.6-openssl
+
+RUN apt-get build-dep -y python3.5
+
+ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256
+
+RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz
+RUN sha256sum -c Python-3.6.0.tar.xz.sha256
+RUN tar xvf Python-3.6.0.tar.xz
+
+ADD myconfigure /Python-3.6.0/myconfigure
+# RUN cd Python-3.6.0/ && ./myconfigure
+# RUN cd Python-3.6.0/ && make -j8 install
+
+ENV PYTHONTARGET=/data/python
 ENV WHEELHOUSE=/data/wheelhouse
 ENV PIP_WHEEL_DIR=/data/wheelhouse
 ENV PIP_FIND_LINKS=/data/wheelhouse
 
 VOLUME /data/wheelhouse
+VOLUME /data/python
+
+# RUN /data/python/bin/python3.6 -m venv /data/venv
+# RUN . /data/venv/bin/activate && pip install -U pip
+# RUN . /data/venv/bin/activate && pip install wheel
 
-ADD requirements.txt /requirements.txt
 ADD build-wheels.sh /build-wheels.sh
 ENTRYPOINT ["bash", "build-wheels.sh"]

docker/2_build/build.sh

@@ -1,11 +1,33 @@
 #!/usr/bin/env bash
 
-mkdir -p ../3_run/wheelhouse;
-cp ../../requirements.txt .;
+# macOS does not support readlink -f, so we use greadlink instead
+if [ $(uname) == 'Darwin' ]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
 
-docker build -t pillar_build -f build.docker .;
-docker run --rm \
-       -v "$(pwd)"/../3_run/wheelhouse:/data/wheelhouse \
-       pillar_build;
+TOPDEVDIR="$($readlink -f ../../..)"
+echo "Top-level development dir is $TOPDEVDIR"
 
-rm requirements.txt;
+PYTHON=$($readlink -f ../3_run/python)
+WHEELHOUSE="$($readlink -f ../3_run/wheelhouse)"
+if [ -z "$WHEELHOUSE" -o -z "$PYTHON" ]; then
+    echo "Error, ../3_run might not exist." >&2
+    exit 2
+fi
+
+mkdir -p "$WHEELHOUSE" "$PYTHON"
+echo "Wheelhouse is $WHEELHOUSE"
+echo "Python will be built to $PYTHON"
+
+docker build -t pillar_build -f build.docker .
+#docker run --rm \
+#    -v "$WHEELHOUSE:/data/wheelhouse" \
+#    -v "$TOPDEVDIR:/data/topdev" \
+#    -v "$PYTHON:/data/python" \
+#    pillar_build
+
+# RUN cd Python-3.6.0/ && ./myconfigure
+# RUN cd Python-3.6.0/ && make -j8 install

docker/2_build/myconfigure

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+./configure \
+    --prefix=$PYTHONTARGET \
+    --enable-ipv6 \
+    --enable-optimizations \
+    --enable-shared \
+    --with-ensurepip=upgrade

docker/3_run/run.docker

@@ -8,6 +8,8 @@ libapache2-mod-wsgi \
 libapache2-mod-xsendfile \
 libjpeg8 \
 libtiff5 \
+ffmpeg \
+rsyslog logrotate \
 nano vim curl \
 && rm -rf /var/lib/apt/lists/*