Explicitly pass hostname to deploy script.

The script now also pings the hostname to deply to, to see if it's alive
before doing anything else.

.gitignore

@@ -12,4 +12,5 @@ __pycache__
 /.eggs/
 /dump/
 /google_app*.json
-/docker/3_run/wheelhouse/
+/docker/2_buildpy/python/
+/docker/4_run/wheelhouse/

README.md

@@ -1,31 +1,32 @@
 # Blender Cloud
 
-Welcome to the [Blender Cloud](https://cloud.blender.org) code repo!
-Blender Cloud runs on the [Pillar](https://pillarframework.org) framework.
+Welcome to the [Blender Cloud](https://cloud.blender.org/) code repo!
+Blender Cloud runs on the [Pillar](https://pillarframework.org/) framework.
 
 ## Quick setup
-Set up a node with these commands. Note that that script is already outdated...
+Set up a node with these commands.
 
 ```
 #!/usr/bin/env bash
 
-mkdir -p /data/git
-mkdir -p /data/storage
-mkdir -p /data/config
-mkdir -p /data/certs
-
+sudo mkdir -p /data/{git,storage,config,certs}
 sudo apt-get update
-sudo apt-get -y install python-pip
-pip install docker-compose
+sudo apt-get -y install python3-pip
+pip3 install docker-compose
 
 cd /data/git
 git clone git://git.blender.org/pillar-python-sdk.git
-git clone git://git.blender.org/pillar-server.git pillar
-git clone git://git.blender.org/attract.git
-git clone git://git.blender.org/flamenco.git
-git clone git://git.blender.org/blender-cloud.git
+git clone git://git.blender.org/pillar.git -b py36
+git clone git://git.blender.org/attract.git -b py36
+git clone git://git.blender.org/flamenco.git -b py36
+git clone git://git.blender.org/blender-cloud.git -b py36
+
 ```
 
+After these commands, run `deploy.sh` to build the static files and deploy
+those too (see below).
+
+
 ## Deploying to production server
 
 First of all, add those aliases to the `[alias]` section of your `~/.gitconfig`
@@ -71,8 +72,8 @@ Now follow the above receipe on the Blender Cloud project as well.
 Contrary to the subprojects, `git pp` will actually perform the deploy
 for real.
 
-Now you can press `[ENTER]` in the Pillar and Attract terminals that
-were still waiting for it.
+Now you can press `[ENTER]` in the Pillar, Attract, and Flamenco terminals
+that were still waiting for it.
 
 After everything is done, your (sub)projects should all be back on
 the master branch.

deploy.sh

@@ -1,11 +1,31 @@
 #!/bin/bash -e
 
+case $1 in
+    cloud*)
+        DEPLOYHOST="$1"
+        ;;
+    *)
+        echo "Use $0 cloud{nr}|cloud.blender.org" >&2
+        exit 1
+esac
+
+echo -n "Deploying to ${DEPLOYHOST}... "
+
+if ! ping ${DEPLOYHOST} -q -c 1 -w 2 >/dev/null; then
+    echo "host ${DEPLOYHOST} cannot be pinged, refusing to deploy." >&2
+    exit 2
+fi
+
+echo "press [ENTER] to continue, Ctrl+C to abort."
+read dummy
+
+
 # Deploys the current production branch to the production machine.
 PROJECT_NAME="blender-cloud"
 DOCKER_NAME="blender_cloud"
 REMOTE_ROOT="/data/git/${PROJECT_NAME}"
 
-SSH="ssh -o ClearAllForwardings=yes cloud.blender.org"
+SSH="ssh -o ClearAllForwardings=yes ${DEPLOYHOST}"
 
 # macOS does not support readlink -f, so we use greadlink instead
 if [[ `uname` == 'Darwin' ]]; then
@@ -91,9 +111,9 @@ git_pull blender-cloud production
 #${SSH} -t docker exec ${DOCKER_NAME} /data/venv/bin/pip install -U -r ${REMOTE_ROOT}/requirements.txt --exists-action w
 
 # RSync the world
-$ATTRACT_DIR/rsync_ui.sh
-$FLAMENCO_DIR/rsync_ui.sh
-./rsync_ui.sh
+$ATTRACT_DIR/rsync_ui.sh ${DEPLOYHOST}
+$FLAMENCO_DIR/rsync_ui.sh ${DEPLOYHOST}
+./rsync_ui.sh ${DEPLOYHOST}
 
 # Notify Bugsnag of this new deploy.
 echo

docker/1_base/base.docker

@@ -1,16 +1,6 @@
-FROM ubuntu:16.04
+FROM ubuntu:16.10
 MAINTAINER Francesco Siddi <francesco@blender.org>
 
 RUN apt-get update && apt-get install -qyy \
     -o APT::Install-Recommends=false -o APT::Install-Suggests=false \
-    python-pip libffi6 openssl ffmpeg rsyslog logrotate
-
-RUN mkdir -p /data/git/pillar \
-&& mkdir -p /data/storage \
-&& mkdir -p /data/config \
-&& mkdir -p /data/venv \
-&& mkdir -p /data/wheelhouse
-
-RUN pip install virtualenv
-RUN virtualenv /data/venv
-RUN . /data/venv/bin/activate && pip install -U pip && pip install wheel
+    openssl ca-certificates

docker/1_base/build.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-docker build -t pillar_base -f base.docker .;
+exec docker build -t pillar_base -f base.docker .

docker/2_build/build-wheels.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-. /data/venv/bin/activate && pip wheel --wheel-dir=/data/wheelhouse -r /requirements.txt

docker/2_build/build.docker

@@ -1,26 +0,0 @@
-FROM pillar_base
-MAINTAINER Francesco Siddi <francesco@blender.org>
-
-RUN apt-get update && apt-get install -qy \
-git \
-gcc \
-libffi-dev \
-libssl-dev \
-pypy-dev \
-python-dev \
-python-imaging \
-zlib1g-dev \
-libjpeg-dev \
-libtiff-dev \
-python-crypto \
-python-openssl
-
-ENV WHEELHOUSE=/data/wheelhouse
-ENV PIP_WHEEL_DIR=/data/wheelhouse
-ENV PIP_FIND_LINKS=/data/wheelhouse
-
-VOLUME /data/wheelhouse
-
-ADD requirements.txt /requirements.txt
-ADD build-wheels.sh /build-wheels.sh
-ENTRYPOINT ["bash", "build-wheels.sh"]

docker/2_build/build.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-mkdir -p ../3_run/wheelhouse;
-cp ../../requirements.txt .;
-
-docker build -t pillar_build -f build.docker .;
-docker run --rm \
-       -v "$(pwd)"/../3_run/wheelhouse:/data/wheelhouse \
-       pillar_build;
-
-rm requirements.txt;

docker/2_buildpy/Python-3.6.0.tar.xz.sha256

@@ -0,0 +1 @@
+b0c5f904f685e32d9232f7bdcbece9819a892929063b6e385414ad2dd6a23622  Python-3.6.0.tar.xz

docker/2_buildpy/build.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -e
+
+# macOS does not support readlink -f, so we use greadlink instead
+if [ $(uname) == 'Darwin' ]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
+
+PYTHONTARGET=$($readlink -f ./python)
+
+mkdir -p "$PYTHONTARGET"
+echo "Python will be built to $PYTHONTARGET"
+
+docker build -t pillar_build -f buildpy.docker .
+
+# Use the docker image to build Python 3.6 and mod-wsgi
+GID=$(id --group)
+docker run --rm -i \
+    -v "$PYTHONTARGET:/opt/python" \
+    pillar_build <<EOT
+set -e
+cd \$PYTHONSOURCE
+./configure \
+    --prefix=/opt/python \
+    --enable-ipv6 \
+    --enable-shared \
+    --with-ensurepip=upgrade
+make -j8 install
+
+# Make sure we can run Python
+ldconfig
+
+# Build mod-wsgi-py3 for Python 3.6
+cd /dpkg/mod-wsgi-*
+./configure --with-python=/opt/python/bin/python3
+make -j8 install
+mkdir -p /opt/python/mod-wsgi
+cp /usr/lib/apache2/modules/mod_wsgi.so /opt/python/mod-wsgi
+
+chown -R $UID:$GID /opt/python/*
+EOT
+
+# Strip some stuff we don't need from the Python install.
+rm -rf $PYTHONTARGET/lib/python3.*/test
+rm -rf $PYTHONTARGET/lib/python3.*/config-3.*/libpython3.*.a
+find $PYTHONTARGET/lib -name '*.so.*' -o -name '*.so' | while read libname; do
+    chmod u+w "$libname"
+    strip "$libname"
+done
+
+# Create another docker image which contains the actual Python.
+# This one will serve as base for the Wheel builder and the
+# production image.
+docker build -t armadillica/pillar_py:3.6 -f includepy.docker .

docker/2_buildpy/buildpy.docker

@@ -0,0 +1,35 @@
+FROM pillar_base
+LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
+
+RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -qy \
+        build-essential \
+        apache2-dev \
+        checkinstall \
+        curl
+
+RUN apt-get build-dep -y python3.5
+
+ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256
+
+# Install Python sources
+RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz && \
+    sha256sum -c Python-3.6.0.tar.xz.sha256 && \
+    tar xf Python-3.6.0.tar.xz && \
+    rm -v Python-3.6.0.tar.xz
+
+# Install mod-wsgi sources
+RUN mkdir -p /dpkg && cd /dpkg && apt-get source libapache2-mod-wsgi-py3
+
+# To be able to install Python outside the docker.
+VOLUME /opt/python
+
+# To be able to run Python; after building, ldconfig has to be re-run to do this.
+# This makes it easier to use Python right after building (for example to build
+# mod-wsgi for Python 3.6).
+RUN echo /opt/python/lib > /etc/ld.so.conf.d/python.conf
+RUN ldconfig
+ENV PATH=/opt/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+ENV PYTHONSOURCE=/Python-3.6.0

docker/2_buildpy/includepy.docker

@@ -0,0 +1,14 @@
+FROM pillar_base
+LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
+
+ADD python /opt/python
+
+RUN echo /opt/python/lib > /etc/ld.so.conf.d/python.conf
+RUN ldconfig
+
+RUN echo Python is installed in /opt/python/ > README.python
+ENV PATH=/opt/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+RUN cd /opt/python/bin && \
+    ln -s python3 python && \
+    ln -s pip3 pip

docker/3_buildwheels/build.docker

@@ -0,0 +1,18 @@
+FROM armadillica/pillar_py:3.6
+LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
+
+RUN apt-get update && apt-get install -qy \
+    git \
+    build-essential \
+    checkinstall \
+    libffi-dev \
+    libssl-dev \
+    libjpeg-dev \
+    zlib1g-dev
+
+ENV WHEELHOUSE=/data/wheelhouse
+ENV PIP_WHEEL_DIR=/data/wheelhouse
+ENV PIP_FIND_LINKS=/data/wheelhouse
+RUN mkdir -p $WHEELHOUSE
+
+VOLUME /data/wheelhouse

docker/3_buildwheels/build.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -e
+
+# macOS does not support readlink -f, so we use greadlink instead
+if [ $(uname) == 'Darwin' ]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
+
+TOPDEVDIR="$($readlink -f ../../..)"
+echo "Top-level development dir is $TOPDEVDIR"
+
+WHEELHOUSE="$($readlink -f ../4_run/wheelhouse)"
+if [ -z "$WHEELHOUSE" ]; then
+    echo "Error, ../4_run might not exist." >&2
+    exit 2
+fi
+
+echo "Wheelhouse is $WHEELHOUSE"
+mkdir -p "$WHEELHOUSE"
+
+docker build -t pillar_wheelbuilder -f build.docker .
+
+GID=$(id --group)
+docker run --rm -i \
+    -v "$WHEELHOUSE:/data/wheelhouse" \
+    -v "$TOPDEVDIR:/data/topdev" \
+    pillar_wheelbuilder <<EOT
+set -e
+# Build wheels for all dependencies.
+cd /data/topdev/blender-cloud
+pip3 install wheel
+pip3 wheel --wheel-dir=/data/wheelhouse -r requirements.txt
+chown -R $UID:$GID /data/wheelhouse
+
+# Install the dependencies so that we can get a full freeze.
+pip3 install --no-index --find-links=/data/wheelhouse -r requirements.txt
+pip3 freeze | grep -v '^-[ef] ' > /data/wheelhouse/requirements.txt
+EOT
+
+# Remove our own projects, they shouldn't be installed as wheel (for now).
+rm -f $WHEELHOUSE/{attract,flamenco,pillar,pillarsdk}*.whl

docker/3_run/build.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-cp ../../requirements.txt .;
-docker build -t armadillica/blender_cloud -f run.docker .;
-rm requirements.txt;

docker/3_run/docker-entrypoint.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-if [ ! -f /installed ]; then
-    echo "Installing pillar and pillar-sdk"
-    # TODO: curretly doing pip install -e takes a long time, so we symlink
-    # . /data/venv/bin/activate && pip install -e /data/git/pillar
-    ln -s /data/git/pillar/pillar /data/venv/lib/python2.7/site-packages/pillar
-    # . /data/venv/bin/activate && pip install -e /data/git/attract
-    ln -s /data/git/attract/attract /data/venv/lib/python2.7/site-packages/attract
-    # . /data/venv/bin/activate && pip install -e /data/git/flamenco/packages/flamenco
-    ln -s /data/git/flamenco/packages/flamenco/flamenco/ /data/venv/lib/python2.7/site-packages/flamenco
-    # . /data/venv/bin/activate && pip install -e /data/git/pillar-python-sdk
-    ln -s /data/git/pillar-python-sdk/pillarsdk /data/venv/lib/python2.7/site-packages/pillarsdk
-    touch installed
-fi
-
-if [ "$DEV" = "true" ]; then
-    echo "Running in development mode"
-    cd /data/git/blender-cloud
-    bash /manage.sh runserver --host='0.0.0.0'
-else
-    # Run Apache
-    a2enmod rewrite
-    /usr/sbin/apache2ctl -D FOREGROUND
-fi

docker/3_run/manage.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash -e
-
-. /data/venv/bin/activate
-cd /data/git/blender-cloud
-python manage.py "$@"

docker/4_run/000-default.conf

@@ -1,10 +1,9 @@
 <VirtualHost *:80>
-	# EnableSendfile on
 	XSendFile on
 	XSendFilePath /data/storage/pillar
 	XSendFilePath /data/git/pillar
-	XSendFilePath /data/venv/lib/python2.7/site-packages/attract/static/
-	XSendFilePath /data/venv/lib/python2.7/site-packages/flamenco/static/
+	XSendFilePath /data/git/attract/attract/static/
+	XSendFilePath /data/git/flamenco/flamenco/static/
 	XsendFilePath /data/git/blender-cloud
 
 	ServerAdmin webmaster@localhost

docker/4_run/bash_history

@@ -0,0 +1,6 @@
+bash docker-entrypoint.sh
+env | sort
+apache2ctl start
+apache2ctl graceful
+tail -n 40 -f /var/log/apache2/access.log
+tail -n 40 -f /var/log/apache2/error.log

docker/4_run/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec docker build -t armadillica/blender_cloud:latest-py36 -f run.docker .

docker/4_run/docker-entrypoint.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+if [ ! -f /installed ]; then
+    SITEPKG=$(echo /opt/python/lib/python3.*/site-packages)
+    echo "Installing Blender Cloud packages into $SITEPKG"
+
+    # TODO: 'pip3 install -e' runs 'setup.py develop', which runs 'setup.py egg_info',
+    # which can't write the egg info to the read-only /data/git volume. This is why
+    # we manually install the links.
+    for SUBPROJ in /data/git/{pillar,pillar-python-sdk,attract,flamenco}; do
+        NAME=$(python3 $SUBPROJ/setup.py --name)
+        echo "... $NAME"
+        echo $SUBPROJ >> $SITEPKG/easy-install.pth
+        echo $SUBPROJ > $SITEPKG/$NAME.egg-link
+    done
+    echo "All packages installed."
+
+    touch /installed
+fi
+
+
+if [ "$DEV" = "true" ]; then
+    echo "Running in development mode"
+    cd /data/git/blender-cloud
+    exec bash /manage.sh runserver --host='0.0.0.0'
+else
+    exec /usr/sbin/apache2ctl -D FOREGROUND
+fi

docker/4_run/manage.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+cd /data/git/blender-cloud
+exec python manage.py "$@"

docker/4_run/run.docker

@@ -1,16 +1,20 @@
-FROM pillar_base
+FROM armadillica/pillar_py:3.6
+LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
 
 RUN apt-get update && apt-get install -qyy \
--o APT::Install-Recommends=true -o APT::Install-Suggests=false \
+-o APT::Install-Recommends=false -o APT::Install-Suggests=false \
 git \
 apache2 \
-libapache2-mod-wsgi \
 libapache2-mod-xsendfile \
 libjpeg8 \
 libtiff5 \
-nano vim curl \
+ffmpeg \
+rsyslog logrotate \
+nano vim-tiny curl \
 && rm -rf /var/lib/apt/lists/*
 
+RUN ln -s /usr/bin/vim.tiny /usr/bin/vim
+
 ENV APACHE_RUN_USER www-data
 ENV APACHE_RUN_GROUP www-data
 ENV APACHE_LOG_DIR /var/log/apache2
@@ -20,16 +24,10 @@ ENV APACHE_LOCK_DIR /var/lock/apache2
 
 RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR
 
-ADD requirements.txt /requirements.txt
 ADD wheelhouse /data/wheelhouse
+RUN pip3 install --no-index --find-links=/data/wheelhouse -r /data/wheelhouse/requirements.txt
 
-RUN . /data/venv/bin/activate \
-&& pip install --no-index --find-links=/data/wheelhouse -r requirements.txt \
-&& rm /requirements.txt
-
-VOLUME /data/git/blender-cloud
-VOLUME /data/git/pillar
-VOLUME /data/git/pillar-python-sdk
+VOLUME /data/git
 VOLUME /data/config
 VOLUME /data/storage
 
@@ -38,9 +36,19 @@ ENV USE_X_SENDFILE True
 EXPOSE 80
 EXPOSE 5000
 
+ADD wsgi-py36.* /etc/apache2/mods-available/
+RUN a2enmod rewrite && a2enmod wsgi-py36
+
 ADD apache2.conf /etc/apache2/apache2.conf
 ADD 000-default.conf /etc/apache2/sites-available/000-default.conf
 ADD docker-entrypoint.sh /docker-entrypoint.sh
 ADD manage.sh /manage.sh
 
-ENTRYPOINT ["bash", "/docker-entrypoint.sh"]
+# Remove some empty top-level directories we won't use anyway.
+RUN rmdir /media /home 2>/dev/null || true
+
+# This file includes some useful commands to have in the shell history
+# for easy access.
+ADD bash_history /root/.bash_history
+
+ENTRYPOINT /docker-entrypoint.sh

docker/4_run/wsgi-py36.conf

@@ -0,0 +1,122 @@
+<IfModule mod_wsgi.c>
+
+
+    #This config file is provided to give an overview of the directives,
+    #which are only allowed in the 'server config' context.
+    #For a detailed description of all avaiable directives please read
+    #http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives
+
+
+    #WSGISocketPrefix: Configure directory to use for daemon sockets.
+    #
+    #Apache's DEFAULT_REL_RUNTIMEDIR should be the proper place for WSGI's
+    #Socket. In case you want to mess with the permissions of the directory,
+    #you need to define WSGISocketPrefix to an alternative directory.
+    #See http://code.google.com/p/modwsgi/wiki/ConfigurationIssues for more
+    #information
+
+        #WSGISocketPrefix /var/run/apache2/wsgi
+
+
+    #WSGIPythonOptimize: Enables basic Python optimisation features.
+    #
+    #Sets the level of Python compiler optimisations. The default is '0'
+    #which means no optimisations are applied.
+    #Setting the optimisation level to '1' or above will have the effect
+    #of enabling basic Python optimisations and changes the filename
+    #extension for compiled (bytecode) files from .pyc to .pyo.
+    #When the optimisation level is set to '2', doc strings will not be
+    #generated and retained. This will result in a smaller memory footprint,
+    #but may cause some Python packages which interrogate doc strings in some
+    #way to fail.
+
+        #WSGIPythonOptimize 0
+
+
+    #WSGIPythonPath: Additional directories to search for Python modules,
+    #                overriding the PYTHONPATH environment variable.
+    #
+    #Used to specify additional directories to search for Python modules.
+    #If multiple directories are specified they should be separated by a ':'.
+
+        WSGIPythonPath /opt/python/lib/python3.6/site-packages
+
+    #WSGIPythonEggs: Directory to use for Python eggs cache.
+    #
+    #Used to specify the directory to be used as the Python eggs cache
+    #directory for all sub interpreters created within embedded mode.
+    #This directive achieves the same affect as having set the
+    #PYTHON_EGG_CACHE environment variable.
+    #Note that the directory specified must exist and be writable by the user
+    #that the Apache child processes run as. The directive only applies to
+    #mod_wsgi embedded mode. To set the Python eggs cache directory for
+    #mod_wsgi daemon processes, use the 'python-eggs' option to the
+    #WSGIDaemonProcess directive instead.
+
+        #WSGIPythonEggs directory
+
+
+
+    #WSGIRestrictEmbedded: Enable restrictions on use of embedded mode.
+    #
+    #The WSGIRestrictEmbedded directive determines whether mod_wsgi embedded
+    #mode is enabled or not. If set to 'On' and the restriction on embedded
+    #mode is therefore enabled, any attempt to make a request against a
+    #WSGI application which hasn't been properly configured so as to be
+    #delegated to a daemon mode process will fail with a HTTP internal server
+    #error response.
+
+        #WSGIRestrictEmbedded On|Off
+
+
+
+    #WSGIRestrictStdin: Enable restrictions on use of STDIN.
+    #WSGIRestrictStdout: Enable restrictions on use of STDOUT.
+    #WSGIRestrictSignal: Enable restrictions on use of signal().
+    #
+    #Well behaved WSGI applications neither should try to read/write from/to
+    #STDIN/STDOUT, nor should they try to register signal handlers. If your
+    #application needs an exception from this rule, you can disable the
+    #restrictions here.
+
+        #WSGIRestrictStdin On
+        #WSGIRestrictStdout On
+        #WSGIRestrictSignal On
+
+
+
+    #WSGIAcceptMutex: Specify type of accept mutex used by daemon processes.
+    #
+    #The WSGIAcceptMutex directive sets the method that mod_wsgi will use to
+    #serialize multiple daemon processes in a process group accepting requests
+    #on a socket connection from the Apache child processes. If this directive
+    #is not defined then the same type of mutex mechanism as used by Apache for
+    #the main Apache child processes when accepting connections from a client
+    #will be used. If set the method types are the same as for the Apache
+    #AcceptMutex directive.
+
+        #WSGIAcceptMutex default
+
+
+
+    #WSGIImportScript: Specify a script file to be loaded on process start.
+    #
+    #The WSGIImportScript directive can be used to specify a script file to be
+    #loaded when a process starts. Options must be provided to indicate the
+    #name of the process group and the application group into which the script
+    #will be loaded.
+
+        #WSGIImportScript process-group=name application-group=name
+
+
+    #WSGILazyInitialization: Enable/disable lazy initialisation of Python.
+    #
+    #The WSGILazyInitialization directives sets whether or not the Python
+    #interpreter is preinitialised within the Apache parent process or whether
+    #lazy initialisation is performed, and the Python interpreter only
+    #initialised in the Apache server processes or mod_wsgi daemon processes
+    #after they have forked from the Apache parent process.
+
+        #WSGILazyInitialization On|Off
+
+</IfModule>

docker/4_run/wsgi-py36.load

@@ -0,0 +1 @@
+LoadModule wsgi_module /opt/python/mod-wsgi/mod_wsgi.so

docker/README.md

@@ -0,0 +1,64 @@
+# Setting up a production machine
+
+To get the docker stack up and running, we use the following, on an Ubuntu 16.10 machine.
+
+## 0. Basic stuff
+
+Install the machine, use `locale-gen nl_NL.UTF-8` or similar commands to generate locale
+definitions. Set up automatic security updates and backups, the usual.
+
+## 1. Install Docker
+
+Install Docker itself, as described in the
+   [Docker CE for Ubuntu manual](https://store.docker.com/editions/community/docker-ce-server-ubuntu?tab=description):
+
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+        $(lsb_release -cs) stable"
+    apt-get update
+    apt-get install docker-ce
+
+## 2. Configure Docker to use "overlay"
+
+Configure Docker to use "overlay" instead of "aufs" for the images. This prevents
+[segfaults in auplink](https://bugs.launchpad.net/ubuntu/+source/aufs-tools/+bug/1442568).
+
+1. Set `DOCKER_OPTS="-s overlay"` in `/etc/defaults/docker`
+2. Edit the `[Service]` section of `/lib/systemd/system/docker.service`:
+    1. Add `EnvironmentFile=/etc/default/docker`
+    2. Append ` $DOCKER_OPTS` to the `ExecStart` line
+3. Run `systemctl daemon-reload`
+4. Remove all your containers and images.
+5. Restart Docker: `systemctl restart docker`
+
+## 3. Pull the Blender Cloud docker image
+
+`docker pull armadillica/blender_cloud:latest-py36`
+
+## 4. Get docker-compose + our repositories
+
+See the [Quick setup](../README.md) on how to get those. Then run:
+
+    cd /data/git/blender-cloud/docker
+    docker-compose up -d
+
+
+Set up permissions for Docker volumes; the following should be writable by
+
+- `/data/storage/pillar`: writable by `www-data` and `root` (do a `chown root:www-data`
+  and `chmod 2770`).
+- `/data/storage/db`: writable by uid 999.
+
+
+## 5. Set up TLS
+
+Place TLS certificates in `/data/certs/{cloud,cloudapi}.blender.org.pem`.
+They should contain (in order) the private key, the host certificate, and the
+CA certificate.
+
+## 6. Create a local config
+
+Blender Cloud expects the following files to exist:
+
+- `/data/git/blender_cloud/config_local.py` with machine-local configuration overrides
+- `/data/config/google_app.json` with Google Cloud Storage credentials.

docker/build.sh

@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 
-set -x;
-set -e;
+set -xe
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd $DIR;
 
-cd 1_base/;
-bash build.sh;
+cd $DIR/1_base
+bash build.sh
 
-cd ../2_build/;
-bash build.sh;
+cd $DIR/2_buildpy
+bash build.sh
 
-cd ../3_run/;
-bash build.sh;
+cd $DIR/3_buildwheels
+bash build.sh
+
+cd $DIR/4_run
+bash build.sh

docker/docker-compose.yml

@@ -1,17 +1,18 @@
 mongo:
-  image: mongo
+  image: mongo:3.4.2
   container_name: mongo
   restart: always
   volumes:
    - /data/storage/db:/data/db
+   - /data/storage/db-bak:/data/db-bak # for backing up stuff etc.
   ports:
    - "127.0.0.1:27017:27017"
 redis:
-  image: redis
+  image: redis:3.2.8
   container_name: redis
   restart: always
 blender_cloud:
-  image: armadillica/blender_cloud
+  image: armadillica/blender_cloud:latest-py36
   container_name: blender_cloud
   restart: always
   environment:
@@ -19,11 +20,7 @@ blender_cloud:
    VIRTUAL_HOST_WEIGHT: 10
    FORCE_SSL: "true"
   volumes:
-   - /data/git/blender-cloud:/data/git/blender-cloud:ro
-   - /data/git/attract:/data/git/attract:ro
-   - /data/git/flamenco:/data/git/flamenco:ro
-   - /data/git/pillar:/data/git/pillar:ro
-   - /data/git/pillar-python-sdk:/data/git/pillar-python-sdk:ro
+   - /data/git:/data/git:ro
    - /data/config:/data/config:ro
    - /data/storage/pillar:/data/storage/pillar
   links:
@@ -40,7 +37,7 @@ blender_cloud:
 #    VIRTUAL_HOST_WEIGHT: 20
 #    FORCE_SSL: true
 grafista:
-  image: armadillica/grafista
+  image: armadillica/grafista:latest
   container_name: grafista
   restart: always
   environment:
@@ -51,7 +48,7 @@ grafista:
     - /data/git/grafista:/data/git/grafista:ro
     - /data/storage/grafista:/data/storage
 haproxy:
-  image: dockercloud/haproxy
+  image: dockercloud/haproxy:1.5.3
   container_name: haproxy
   restart: always
   ports:

manage.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
-
 import logging
 from flask import current_app
 from pillar import cli

requirements-dev.txt

@@ -0,0 +1,9 @@
+-r ../pillar-python-sdk/requirements-dev.txt
+-r ../pillar/requirements-dev.txt
+-r ../attract/requirements-dev.txt
+-r ../flamenco/requirements-dev.txt
+
+-e ../pillar-python-sdk
+-e ../pillar
+-e ../attract
+-e ../flamenco

requirements.txt

@@ -1,67 +1,3 @@
-# Primary requirements
-# pillarsdk
-# pillar
-# attract
-# flamenco
-
-# Secondary requirements (i.e. pulled in from primary requirements)
-algoliasearch==1.8.0
-attrs==16.2.0
-bcrypt==2.0.0
-blinker==1.4
-bugsnag==2.3.1
-bleach==1.4.3
-Cerberus==0.9.2
-cffi==1.7.0
-commonmark==0.7.2
-cryptography==1.4
-enum34==1.1.6
-Eve==0.6.3
-Events==0.2.1
-Flask==0.10.1
-Flask-Cache==0.13.1
-Flask-Script==2.0.5
-Flask-Login==0.3.2
-Flask-OAuthlib==0.9.3
-Flask-PyMongo==0.4.1
-Flask-WTF==0.12
-flup==1.0.2
-future==0.15.2
-gcloud==0.12.0
-google-apitools==0.4.11
-googleapis-common-protos==1.2.0
-html5lib==0.9999999
-httplib2==0.9.2
-idna==2.0
-ipaddress==1.0.16
-itsdangerous==0.24
-Jinja2==2.8
-MarkupSafe==0.23
-markdown==2.6.7
-ndg-httpsclient==0.4.0
-oauth2client==3.0.0
-oauthlib==1.1.2
-pathlib2==2.2.1
-Pillow==2.8.1
-protobuf==3.0.0
-protorpc==0.11.1
-pyasn1==0.1.9
-pyasn1-modules==0.0.8
-pycparser==2.14
-pycrypto==2.6.1
-pylru==1.0.4
-pymongo==3.3.0
-pyOpenSSL==0.15.1
-python-dateutil==2.5.3
-redis==2.10.5
-requests==2.9.1
-requests-oauthlib==0.6.2
-rsa==3.4.2
-scandir==1.4
-simplejson==3.8.2
-six==1.10.0
-svn==0.3.43
-WebOb==1.5.0
-Werkzeug==0.11.10
-WTForms==2.1
-zencoder==0.6.5
+-r ../pillar/requirements.txt
+-r ../attract/requirements.txt
+-r ../flamenco/requirements.txt

rsync_ui.sh

@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
 
+set -e  # error out when one of the commands in the script errors.
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 {host-to-deploy-to}" >&2
+    exit 1
+fi
+
+DEPLOYHOST="$1"
+
 PILLAR_DIR=$(python <<EOT
 from __future__ import print_function
 import os.path
@@ -33,8 +42,8 @@ fi
 
 echo
 echo "*** SYNCING ASSETS ***"
-rsync -avh $ASSETS root@cloud.blender.org:/data/git/pillar/pillar/web/static/assets/
+rsync -avh $ASSETS root@${DEPLOYHOST}:/data/git/pillar/pillar/web/static/assets/
 
 echo
 echo "*** SYNCING TEMPLATES ***"
-rsync -avh $TEMPLATES root@cloud.blender.org:/data/git/pillar/pillar/web/templates/
+rsync -avh $TEMPLATES root@${DEPLOYHOST}:/data/git/pillar/pillar/web/templates/

runserver.wsgi

@@ -1,14 +1,11 @@
 from os.path import abspath, dirname
 import sys
 
-activate_this = '/data/venv/bin/activate_this.py'
-execfile(activate_this, dict(__file__=activate_this))
-from flup.server.fcgi import WSGIServer
 from pillar import PillarServer
 from attract import AttractExtension
 from flamenco import FlamencoExtension
 
-sys.path.append('/data/git/blender-cloud/')
+sys.path.append('/data/git/blender-cloud')
 
 attract = AttractExtension()
 flamenco = FlamencoExtension()
@@ -17,6 +14,3 @@ application = PillarServer(dirname(abspath(__file__)))
 application.load_extension(attract, '/attract')
 application.load_extension(flamenco, '/flamenco')
 application.process_extensions()
-
-if __name__ == '__main__':
-    WSGIServer(application).run()