WIP instructions for dev installation from scratch

.gitignore

@@ -12,5 +12,4 @@ __pycache__
 /.eggs/
 /dump/
 /google_app*.json
-/docker/2_buildpy/python/
+/docker/3_run/wheelhouse/
-/docker/4_run/wheelhouse/

README.md

@@ -1,32 +1,31 @@
 # Blender Cloud
 
-Welcome to the [Blender Cloud](https://cloud.blender.org/) code repo!
+Welcome to the [Blender Cloud](https://cloud.blender.org) code repo!
-Blender Cloud runs on the [Pillar](https://pillarframework.org/) framework.
+Blender Cloud runs on the [Pillar](https://pillarframework.org) framework.
 
 ## Quick setup
-Set up a node with these commands.
+Set up a node with these commands. Note that that script is already outdated...
 
 ```
 #!/usr/bin/env bash
 
-sudo mkdir -p /data/{git,storage,config,certs}
+mkdir -p /data/git
+mkdir -p /data/storage
+mkdir -p /data/config
+mkdir -p /data/certs
+
 sudo apt-get update
-sudo apt-get -y install python3-pip
+sudo apt-get -y install python-pip
-pip3 install docker-compose
+pip install docker-compose
 
 cd /data/git
 git clone git://git.blender.org/pillar-python-sdk.git
-git clone git://git.blender.org/pillar.git -b py36
+git clone git://git.blender.org/pillar-server.git pillar
-git clone git://git.blender.org/attract.git -b py36
+git clone git://git.blender.org/attract.git
-git clone git://git.blender.org/flamenco.git -b py36
+git clone git://git.blender.org/flamenco.git
-git clone git://git.blender.org/blender-cloud.git -b py36
+git clone git://git.blender.org/blender-cloud.git
-
 ```
 
-After these commands, run `deploy.sh` to build the static files and deploy
-those too (see below).
-
-
 ## Deploying to production server
 
 First of all, add those aliases to the `[alias]` section of your `~/.gitconfig`
@@ -72,8 +71,8 @@ Now follow the above receipe on the Blender Cloud project as well.
 Contrary to the subprojects, `git pp` will actually perform the deploy
 for real.
 
-Now you can press `[ENTER]` in the Pillar, Attract, and Flamenco terminals
+Now you can press `[ENTER]` in the Pillar and Attract terminals that
-that were still waiting for it.
+were still waiting for it.
 
 After everything is done, your (sub)projects should all be back on
 the master branch.
@@ -93,3 +92,27 @@ On the production machine:
 
 1. `docker pull armadillica/blender_cloud`
 2. `docker-compose up -d` (from the `/data/git/blender-cloud/docker` directory)
+
+
+## Development setup
+
+Here are some notes on how to get the whole Blender Cloud dev environment set up
+from scratch. First off, we agree to develop on bare metal and use Docker only for
+production.
+
+Initial requirements:
+
+- Python 2.7
+- NPM
+- Blender ID server up and running (see blender-id for more)
+
+Here are the steps:
+
+- Clone the fantastic 5 (pillar, pillar-python-sdk, attract, flamenco and 
+blender-cloud)
+- Run `npm install` in pillar, attract and flamenco
+- Create `config_local.py` in blender-cloud
+- `pip install -e .` in every repo except blender-cloud
+- `pip install -r requirements.txt` in blender-cloud
+- setup_db
+- create_urler_account

deploy.sh

@@ -1,31 +1,11 @@
 #!/bin/bash -e
 
-case $1 in
-    cloud*)
-        DEPLOYHOST="$1"
-        ;;
-    *)
-        echo "Use $0 cloud{nr}|cloud.blender.org" >&2
-        exit 1
-esac
-
-echo -n "Deploying to ${DEPLOYHOST}... "
-
-if ! ping ${DEPLOYHOST} -q -c 1 -w 2 >/dev/null; then
-    echo "host ${DEPLOYHOST} cannot be pinged, refusing to deploy." >&2
-    exit 2
-fi
-
-echo "press [ENTER] to continue, Ctrl+C to abort."
-read dummy
-
-
 # Deploys the current production branch to the production machine.
 PROJECT_NAME="blender-cloud"
 DOCKER_NAME="blender_cloud"
 REMOTE_ROOT="/data/git/${PROJECT_NAME}"
 
-SSH="ssh -o ClearAllForwardings=yes ${DEPLOYHOST}"
+SSH="ssh -o ClearAllForwardings=yes cloud.blender.org"
 
 # macOS does not support readlink -f, so we use greadlink instead
 if [[ `uname` == 'Darwin' ]]; then
@@ -111,9 +91,9 @@ git_pull blender-cloud production
 #${SSH} -t docker exec ${DOCKER_NAME} /data/venv/bin/pip install -U -r ${REMOTE_ROOT}/requirements.txt --exists-action w
 
 # RSync the world
-$ATTRACT_DIR/rsync_ui.sh ${DEPLOYHOST}
+$ATTRACT_DIR/rsync_ui.sh
-$FLAMENCO_DIR/rsync_ui.sh ${DEPLOYHOST}
+$FLAMENCO_DIR/rsync_ui.sh
-./rsync_ui.sh ${DEPLOYHOST}
+./rsync_ui.sh
 
 # Notify Bugsnag of this new deploy.
 echo

docker/1_base/base.docker

@@ -1,6 +1,16 @@
-FROM ubuntu:16.10
+FROM ubuntu:16.04
 MAINTAINER Francesco Siddi <francesco@blender.org>
 
 RUN apt-get update && apt-get install -qyy \
     -o APT::Install-Recommends=false -o APT::Install-Suggests=false \
-    openssl ca-certificates
+    python-pip libffi6 openssl ffmpeg rsyslog logrotate
+
+RUN mkdir -p /data/git/pillar \
+&& mkdir -p /data/storage \
+&& mkdir -p /data/config \
+&& mkdir -p /data/venv \
+&& mkdir -p /data/wheelhouse
+
+RUN pip install virtualenv
+RUN virtualenv /data/venv
+RUN . /data/venv/bin/activate && pip install -U pip && pip install wheel

docker/1_base/build.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-exec docker build -t pillar_base -f base.docker .
+docker build -t pillar_base -f base.docker .;

docker/2_build/build-wheels.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+. /data/venv/bin/activate && pip wheel --wheel-dir=/data/wheelhouse -r /requirements.txt

docker/2_build/build.docker

@@ -0,0 +1,26 @@
+FROM pillar_base
+MAINTAINER Francesco Siddi <francesco@blender.org>
+
+RUN apt-get update && apt-get install -qy \
+git \
+gcc \
+libffi-dev \
+libssl-dev \
+pypy-dev \
+python-dev \
+python-imaging \
+zlib1g-dev \
+libjpeg-dev \
+libtiff-dev \
+python-crypto \
+python-openssl
+
+ENV WHEELHOUSE=/data/wheelhouse
+ENV PIP_WHEEL_DIR=/data/wheelhouse
+ENV PIP_FIND_LINKS=/data/wheelhouse
+
+VOLUME /data/wheelhouse
+
+ADD requirements.txt /requirements.txt
+ADD build-wheels.sh /build-wheels.sh
+ENTRYPOINT ["bash", "build-wheels.sh"]

docker/2_build/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+mkdir -p ../3_run/wheelhouse;
+cp ../../requirements.txt .;
+
+docker build -t pillar_build -f build.docker .;
+docker run --rm \
+       -v "$(pwd)"/../3_run/wheelhouse:/data/wheelhouse \
+       pillar_build;
+
+rm requirements.txt;

docker/2_buildpy/Python-3.6.0.tar.xz.sha256

@@ -1 +0,0 @@
-b0c5f904f685e32d9232f7bdcbece9819a892929063b6e385414ad2dd6a23622  Python-3.6.0.tar.xz

docker/2_buildpy/build.sh

@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# macOS does not support readlink -f, so we use greadlink instead
-if [ $(uname) == 'Darwin' ]; then
-    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
-    readlink='greadlink'
-else
-    readlink='readlink'
-fi
-
-PYTHONTARGET=$($readlink -f ./python)
-
-mkdir -p "$PYTHONTARGET"
-echo "Python will be built to $PYTHONTARGET"
-
-docker build -t pillar_build -f buildpy.docker .
-
-# Use the docker image to build Python 3.6 and mod-wsgi
-GID=$(id --group)
-docker run --rm -i \
-    -v "$PYTHONTARGET:/opt/python" \
-    pillar_build <<EOT
-set -e
-cd \$PYTHONSOURCE
-./configure \
-    --prefix=/opt/python \
-    --enable-ipv6 \
-    --enable-shared \
-    --with-ensurepip=upgrade
-make -j8 install
-
-# Make sure we can run Python
-ldconfig
-
-# Build mod-wsgi-py3 for Python 3.6
-cd /dpkg/mod-wsgi-*
-./configure --with-python=/opt/python/bin/python3
-make -j8 install
-mkdir -p /opt/python/mod-wsgi
-cp /usr/lib/apache2/modules/mod_wsgi.so /opt/python/mod-wsgi
-
-chown -R $UID:$GID /opt/python/*
-EOT
-
-# Strip some stuff we don't need from the Python install.
-rm -rf $PYTHONTARGET/lib/python3.*/test
-rm -rf $PYTHONTARGET/lib/python3.*/config-3.*/libpython3.*.a
-find $PYTHONTARGET/lib -name '*.so.*' -o -name '*.so' | while read libname; do
-    chmod u+w "$libname"
-    strip "$libname"
-done
-
-# Create another docker image which contains the actual Python.
-# This one will serve as base for the Wheel builder and the
-# production image.
-docker build -t armadillica/pillar_py:3.6 -f includepy.docker .

docker/2_buildpy/buildpy.docker

@@ -1,35 +0,0 @@
-FROM pillar_base
-LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
-
-RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -qy \
-        build-essential \
-        apache2-dev \
-        checkinstall \
-        curl
-
-RUN apt-get build-dep -y python3.5
-
-ADD Python-3.6.0.tar.xz.sha256 /Python-3.6.0.tar.xz.sha256
-
-# Install Python sources
-RUN curl -O https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz && \
-    sha256sum -c Python-3.6.0.tar.xz.sha256 && \
-    tar xf Python-3.6.0.tar.xz && \
-    rm -v Python-3.6.0.tar.xz
-
-# Install mod-wsgi sources
-RUN mkdir -p /dpkg && cd /dpkg && apt-get source libapache2-mod-wsgi-py3
-
-# To be able to install Python outside the docker.
-VOLUME /opt/python
-
-# To be able to run Python; after building, ldconfig has to be re-run to do this.
-# This makes it easier to use Python right after building (for example to build
-# mod-wsgi for Python 3.6).
-RUN echo /opt/python/lib > /etc/ld.so.conf.d/python.conf
-RUN ldconfig
-ENV PATH=/opt/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-ENV PYTHONSOURCE=/Python-3.6.0

docker/2_buildpy/includepy.docker

@@ -1,14 +0,0 @@
-FROM pillar_base
-LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
-
-ADD python /opt/python
-
-RUN echo /opt/python/lib > /etc/ld.so.conf.d/python.conf
-RUN ldconfig
-
-RUN echo Python is installed in /opt/python/ > README.python
-ENV PATH=/opt/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-RUN cd /opt/python/bin && \
-    ln -s python3 python && \
-    ln -s pip3 pip

docker/3_buildwheels/build.docker

@@ -1,18 +0,0 @@
-FROM armadillica/pillar_py:3.6
-LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
-
-RUN apt-get update && apt-get install -qy \
-    git \
-    build-essential \
-    checkinstall \
-    libffi-dev \
-    libssl-dev \
-    libjpeg-dev \
-    zlib1g-dev
-
-ENV WHEELHOUSE=/data/wheelhouse
-ENV PIP_WHEEL_DIR=/data/wheelhouse
-ENV PIP_FIND_LINKS=/data/wheelhouse
-RUN mkdir -p $WHEELHOUSE
-
-VOLUME /data/wheelhouse

docker/3_buildwheels/build.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# macOS does not support readlink -f, so we use greadlink instead
-if [ $(uname) == 'Darwin' ]; then
-    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
-    readlink='greadlink'
-else
-    readlink='readlink'
-fi
-
-TOPDEVDIR="$($readlink -f ../../..)"
-echo "Top-level development dir is $TOPDEVDIR"
-
-WHEELHOUSE="$($readlink -f ../4_run/wheelhouse)"
-if [ -z "$WHEELHOUSE" ]; then
-    echo "Error, ../4_run might not exist." >&2
-    exit 2
-fi
-
-echo "Wheelhouse is $WHEELHOUSE"
-mkdir -p "$WHEELHOUSE"
-
-docker build -t pillar_wheelbuilder -f build.docker .
-
-GID=$(id --group)
-docker run --rm -i \
-    -v "$WHEELHOUSE:/data/wheelhouse" \
-    -v "$TOPDEVDIR:/data/topdev" \
-    pillar_wheelbuilder <<EOT
-set -e
-# Build wheels for all dependencies.
-cd /data/topdev/blender-cloud
-pip3 install wheel
-pip3 wheel --wheel-dir=/data/wheelhouse -r requirements.txt
-chown -R $UID:$GID /data/wheelhouse
-
-# Install the dependencies so that we can get a full freeze.
-pip3 install --no-index --find-links=/data/wheelhouse -r requirements.txt
-pip3 freeze | grep -v '^-[ef] ' > /data/wheelhouse/requirements.txt
-EOT
-
-# Remove our own projects, they shouldn't be installed as wheel (for now).
-rm -f $WHEELHOUSE/{attract,flamenco,pillar,pillarsdk}*.whl

docker/3_run/000-default.conf

@@ -1,9 +1,10 @@
 <VirtualHost *:80>
+	# EnableSendfile on
 	XSendFile on
 	XSendFilePath /data/storage/pillar
 	XSendFilePath /data/git/pillar
-	XSendFilePath /data/git/attract/attract/static/
+	XSendFilePath /data/venv/lib/python2.7/site-packages/attract/static/
-	XSendFilePath /data/git/flamenco/flamenco/static/
+	XSendFilePath /data/venv/lib/python2.7/site-packages/flamenco/static/
 	XsendFilePath /data/git/blender-cloud
 
 	ServerAdmin webmaster@localhost

docker/3_run/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cp ../../requirements.txt .;
+docker build -t armadillica/blender_cloud -f run.docker .;
+rm requirements.txt;

docker/3_run/docker-entrypoint.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+if [ ! -f /installed ]; then
+    echo "Installing pillar and pillar-sdk"
+    # TODO: curretly doing pip install -e takes a long time, so we symlink
+    # . /data/venv/bin/activate && pip install -e /data/git/pillar
+    ln -s /data/git/pillar/pillar /data/venv/lib/python2.7/site-packages/pillar
+    # . /data/venv/bin/activate && pip install -e /data/git/attract
+    ln -s /data/git/attract/attract /data/venv/lib/python2.7/site-packages/attract
+    # . /data/venv/bin/activate && pip install -e /data/git/flamenco/packages/flamenco
+    ln -s /data/git/flamenco/packages/flamenco/flamenco/ /data/venv/lib/python2.7/site-packages/flamenco
+    # . /data/venv/bin/activate && pip install -e /data/git/pillar-python-sdk
+    ln -s /data/git/pillar-python-sdk/pillarsdk /data/venv/lib/python2.7/site-packages/pillarsdk
+    touch installed
+fi
+
+if [ "$DEV" = "true" ]; then
+    echo "Running in development mode"
+    cd /data/git/blender-cloud
+    bash /manage.sh runserver --host='0.0.0.0'
+else
+    # Run Apache
+    a2enmod rewrite
+    /usr/sbin/apache2ctl -D FOREGROUND
+fi

docker/3_run/manage.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash -e
+
+. /data/venv/bin/activate
+cd /data/git/blender-cloud
+python manage.py "$@"

docker/3_run/run.docker

@@ -1,20 +1,16 @@
-FROM armadillica/pillar_py:3.6
+FROM pillar_base
-LABEL maintainer Sybren A. Stüvel <sybren@blender.studio>
 
 RUN apt-get update && apt-get install -qyy \
--o APT::Install-Recommends=false -o APT::Install-Suggests=false \
+-o APT::Install-Recommends=true -o APT::Install-Suggests=false \
 git \
 apache2 \
+libapache2-mod-wsgi \
 libapache2-mod-xsendfile \
 libjpeg8 \
 libtiff5 \
-ffmpeg \
+nano vim curl \
-rsyslog logrotate \
-nano vim-tiny curl \
 && rm -rf /var/lib/apt/lists/*
 
-RUN ln -s /usr/bin/vim.tiny /usr/bin/vim
-
 ENV APACHE_RUN_USER www-data
 ENV APACHE_RUN_GROUP www-data
 ENV APACHE_LOG_DIR /var/log/apache2
@@ -24,10 +20,16 @@ ENV APACHE_LOCK_DIR /var/lock/apache2
 
 RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR
 
+ADD requirements.txt /requirements.txt
 ADD wheelhouse /data/wheelhouse
-RUN pip3 install --no-index --find-links=/data/wheelhouse -r /data/wheelhouse/requirements.txt
 
-VOLUME /data/git
+RUN . /data/venv/bin/activate \
+&& pip install --no-index --find-links=/data/wheelhouse -r requirements.txt \
+&& rm /requirements.txt
+
+VOLUME /data/git/blender-cloud
+VOLUME /data/git/pillar
+VOLUME /data/git/pillar-python-sdk
 VOLUME /data/config
 VOLUME /data/storage
 
@@ -36,19 +38,9 @@ ENV USE_X_SENDFILE True
 EXPOSE 80
 EXPOSE 5000
 
-ADD wsgi-py36.* /etc/apache2/mods-available/
-RUN a2enmod rewrite && a2enmod wsgi-py36
-
 ADD apache2.conf /etc/apache2/apache2.conf
 ADD 000-default.conf /etc/apache2/sites-available/000-default.conf
 ADD docker-entrypoint.sh /docker-entrypoint.sh
 ADD manage.sh /manage.sh
 
-# Remove some empty top-level directories we won't use anyway.
+ENTRYPOINT ["bash", "/docker-entrypoint.sh"]
-RUN rmdir /media /home 2>/dev/null || true
-
-# This file includes some useful commands to have in the shell history
-# for easy access.
-ADD bash_history /root/.bash_history
-
-ENTRYPOINT /docker-entrypoint.sh

docker/4_run/bash_history

@@ -1,6 +0,0 @@
-bash docker-entrypoint.sh
-env | sort
-apache2ctl start
-apache2ctl graceful
-tail -n 40 -f /var/log/apache2/access.log
-tail -n 40 -f /var/log/apache2/error.log

docker/4_run/build.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-exec docker build -t armadillica/blender_cloud:latest-py36 -f run.docker .

docker/4_run/docker-entrypoint.sh

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-if [ ! -f /installed ]; then
-    SITEPKG=$(echo /opt/python/lib/python3.*/site-packages)
-    echo "Installing Blender Cloud packages into $SITEPKG"
-
-    # TODO: 'pip3 install -e' runs 'setup.py develop', which runs 'setup.py egg_info',
-    # which can't write the egg info to the read-only /data/git volume. This is why
-    # we manually install the links.
-    for SUBPROJ in /data/git/{pillar,pillar-python-sdk,attract,flamenco}; do
-        NAME=$(python3 $SUBPROJ/setup.py --name)
-        echo "... $NAME"
-        echo $SUBPROJ >> $SITEPKG/easy-install.pth
-        echo $SUBPROJ > $SITEPKG/$NAME.egg-link
-    done
-    echo "All packages installed."
-
-    touch /installed
-fi
-
-
-if [ "$DEV" = "true" ]; then
-    echo "Running in development mode"
-    cd /data/git/blender-cloud
-    exec bash /manage.sh runserver --host='0.0.0.0'
-else
-    exec /usr/sbin/apache2ctl -D FOREGROUND
-fi

docker/4_run/manage.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-cd /data/git/blender-cloud
-exec python manage.py "$@"

docker/4_run/wsgi-py36.conf

@@ -1,122 +0,0 @@
-<IfModule mod_wsgi.c>
-
-
-    #This config file is provided to give an overview of the directives,
-    #which are only allowed in the 'server config' context.
-    #For a detailed description of all avaiable directives please read
-    #http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives
-
-
-    #WSGISocketPrefix: Configure directory to use for daemon sockets.
-    #
-    #Apache's DEFAULT_REL_RUNTIMEDIR should be the proper place for WSGI's
-    #Socket. In case you want to mess with the permissions of the directory,
-    #you need to define WSGISocketPrefix to an alternative directory.
-    #See http://code.google.com/p/modwsgi/wiki/ConfigurationIssues for more
-    #information
-
-        #WSGISocketPrefix /var/run/apache2/wsgi
-
-
-    #WSGIPythonOptimize: Enables basic Python optimisation features.
-    #
-    #Sets the level of Python compiler optimisations. The default is '0'
-    #which means no optimisations are applied.
-    #Setting the optimisation level to '1' or above will have the effect
-    #of enabling basic Python optimisations and changes the filename
-    #extension for compiled (bytecode) files from .pyc to .pyo.
-    #When the optimisation level is set to '2', doc strings will not be
-    #generated and retained. This will result in a smaller memory footprint,
-    #but may cause some Python packages which interrogate doc strings in some
-    #way to fail.
-
-        #WSGIPythonOptimize 0
-
-
-    #WSGIPythonPath: Additional directories to search for Python modules,
-    #                overriding the PYTHONPATH environment variable.
-    #
-    #Used to specify additional directories to search for Python modules.
-    #If multiple directories are specified they should be separated by a ':'.
-
-        WSGIPythonPath /opt/python/lib/python3.6/site-packages
-
-    #WSGIPythonEggs: Directory to use for Python eggs cache.
-    #
-    #Used to specify the directory to be used as the Python eggs cache
-    #directory for all sub interpreters created within embedded mode.
-    #This directive achieves the same affect as having set the
-    #PYTHON_EGG_CACHE environment variable.
-    #Note that the directory specified must exist and be writable by the user
-    #that the Apache child processes run as. The directive only applies to
-    #mod_wsgi embedded mode. To set the Python eggs cache directory for
-    #mod_wsgi daemon processes, use the 'python-eggs' option to the
-    #WSGIDaemonProcess directive instead.
-
-        #WSGIPythonEggs directory
-
-
-
-    #WSGIRestrictEmbedded: Enable restrictions on use of embedded mode.
-    #
-    #The WSGIRestrictEmbedded directive determines whether mod_wsgi embedded
-    #mode is enabled or not. If set to 'On' and the restriction on embedded
-    #mode is therefore enabled, any attempt to make a request against a
-    #WSGI application which hasn't been properly configured so as to be
-    #delegated to a daemon mode process will fail with a HTTP internal server
-    #error response.
-
-        #WSGIRestrictEmbedded On|Off
-
-
-
-    #WSGIRestrictStdin: Enable restrictions on use of STDIN.
-    #WSGIRestrictStdout: Enable restrictions on use of STDOUT.
-    #WSGIRestrictSignal: Enable restrictions on use of signal().
-    #
-    #Well behaved WSGI applications neither should try to read/write from/to
-    #STDIN/STDOUT, nor should they try to register signal handlers. If your
-    #application needs an exception from this rule, you can disable the
-    #restrictions here.
-
-        #WSGIRestrictStdin On
-        #WSGIRestrictStdout On
-        #WSGIRestrictSignal On
-
-
-
-    #WSGIAcceptMutex: Specify type of accept mutex used by daemon processes.
-    #
-    #The WSGIAcceptMutex directive sets the method that mod_wsgi will use to
-    #serialize multiple daemon processes in a process group accepting requests
-    #on a socket connection from the Apache child processes. If this directive
-    #is not defined then the same type of mutex mechanism as used by Apache for
-    #the main Apache child processes when accepting connections from a client
-    #will be used. If set the method types are the same as for the Apache
-    #AcceptMutex directive.
-
-        #WSGIAcceptMutex default
-
-
-
-    #WSGIImportScript: Specify a script file to be loaded on process start.
-    #
-    #The WSGIImportScript directive can be used to specify a script file to be
-    #loaded when a process starts. Options must be provided to indicate the
-    #name of the process group and the application group into which the script
-    #will be loaded.
-
-        #WSGIImportScript process-group=name application-group=name
-
-
-    #WSGILazyInitialization: Enable/disable lazy initialisation of Python.
-    #
-    #The WSGILazyInitialization directives sets whether or not the Python
-    #interpreter is preinitialised within the Apache parent process or whether
-    #lazy initialisation is performed, and the Python interpreter only
-    #initialised in the Apache server processes or mod_wsgi daemon processes
-    #after they have forked from the Apache parent process.
-
-        #WSGILazyInitialization On|Off
-
-</IfModule>

docker/4_run/wsgi-py36.load

@@ -1 +0,0 @@
-LoadModule wsgi_module /opt/python/mod-wsgi/mod_wsgi.so

docker/README.md

@@ -1,64 +0,0 @@
-# Setting up a production machine
-
-To get the docker stack up and running, we use the following, on an Ubuntu 16.10 machine.
-
-## 0. Basic stuff
-
-Install the machine, use `locale-gen nl_NL.UTF-8` or similar commands to generate locale
-definitions. Set up automatic security updates and backups, the usual.
-
-## 1. Install Docker
-
-Install Docker itself, as described in the
-   [Docker CE for Ubuntu manual](https://store.docker.com/editions/community/docker-ce-server-ubuntu?tab=description):
-
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
-        $(lsb_release -cs) stable"
-    apt-get update
-    apt-get install docker-ce
-
-## 2. Configure Docker to use "overlay"
-
-Configure Docker to use "overlay" instead of "aufs" for the images. This prevents
-[segfaults in auplink](https://bugs.launchpad.net/ubuntu/+source/aufs-tools/+bug/1442568).
-
-1. Set `DOCKER_OPTS="-s overlay"` in `/etc/defaults/docker`
-2. Edit the `[Service]` section of `/lib/systemd/system/docker.service`:
-    1. Add `EnvironmentFile=/etc/default/docker`
-    2. Append ` $DOCKER_OPTS` to the `ExecStart` line
-3. Run `systemctl daemon-reload`
-4. Remove all your containers and images.
-5. Restart Docker: `systemctl restart docker`
-
-## 3. Pull the Blender Cloud docker image
-
-`docker pull armadillica/blender_cloud:latest-py36`
-
-## 4. Get docker-compose + our repositories
-
-See the [Quick setup](../README.md) on how to get those. Then run:
-
-    cd /data/git/blender-cloud/docker
-    docker-compose up -d
-
-
-Set up permissions for Docker volumes; the following should be writable by
-
-- `/data/storage/pillar`: writable by `www-data` and `root` (do a `chown root:www-data`
-  and `chmod 2770`).
-- `/data/storage/db`: writable by uid 999.
-
-
-## 5. Set up TLS
-
-Place TLS certificates in `/data/certs/{cloud,cloudapi}.blender.org.pem`.
-They should contain (in order) the private key, the host certificate, and the
-CA certificate.
-
-## 6. Create a local config
-
-Blender Cloud expects the following files to exist:
-
-- `/data/git/blender_cloud/config_local.py` with machine-local configuration overrides
-- `/data/config/google_app.json` with Google Cloud Storage credentials.

docker/build.sh

@@ -1,17 +1,16 @@
 #!/usr/bin/env bash
 
-set -xe
+set -x;
+set -e;
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $DIR;
 
-cd $DIR/1_base
+cd 1_base/;
-bash build.sh
+bash build.sh;
 
-cd $DIR/2_buildpy
+cd ../2_build/;
-bash build.sh
+bash build.sh;
 
-cd $DIR/3_buildwheels
+cd ../3_run/;
-bash build.sh
+bash build.sh;
-
-cd $DIR/4_run
-bash build.sh

docker/docker-compose.yml

@@ -1,18 +1,17 @@
 mongo:
-  image: mongo:3.4.2
+  image: mongo
   container_name: mongo
   restart: always
   volumes:
    - /data/storage/db:/data/db
-   - /data/storage/db-bak:/data/db-bak # for backing up stuff etc.
   ports:
    - "127.0.0.1:27017:27017"
 redis:
-  image: redis:3.2.8
+  image: redis
   container_name: redis
   restart: always
 blender_cloud:
-  image: armadillica/blender_cloud:latest-py36
+  image: armadillica/blender_cloud
   container_name: blender_cloud
   restart: always
   environment:
@@ -20,7 +19,11 @@ blender_cloud:
    VIRTUAL_HOST_WEIGHT: 10
    FORCE_SSL: "true"
   volumes:
-   - /data/git:/data/git:ro
+   - /data/git/blender-cloud:/data/git/blender-cloud:ro
+   - /data/git/attract:/data/git/attract:ro
+   - /data/git/flamenco:/data/git/flamenco:ro
+   - /data/git/pillar:/data/git/pillar:ro
+   - /data/git/pillar-python-sdk:/data/git/pillar-python-sdk:ro
    - /data/config:/data/config:ro
    - /data/storage/pillar:/data/storage/pillar
   links:
@@ -37,7 +40,7 @@ blender_cloud:
 #    VIRTUAL_HOST_WEIGHT: 20
 #    FORCE_SSL: true
 grafista:
-  image: armadillica/grafista:latest
+  image: armadillica/grafista
   container_name: grafista
   restart: always
   environment:
@@ -48,7 +51,7 @@ grafista:
     - /data/git/grafista:/data/git/grafista:ro
     - /data/storage/grafista:/data/storage
 haproxy:
-  image: dockercloud/haproxy:1.5.3
+  image: dockercloud/haproxy
   container_name: haproxy
   restart: always
   ports:

manage.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import logging
 from flask import current_app
 from pillar import cli

requirements-dev.txt

@@ -1,9 +0,0 @@
--r ../pillar-python-sdk/requirements-dev.txt
--r ../pillar/requirements-dev.txt
--r ../attract/requirements-dev.txt
--r ../flamenco/requirements-dev.txt
-
--e ../pillar-python-sdk
--e ../pillar
--e ../attract
--e ../flamenco

requirements.txt

@@ -1,3 +1,67 @@
--r ../pillar/requirements.txt
+# Primary requirements
--r ../attract/requirements.txt
+# pillarsdk
--r ../flamenco/requirements.txt
+# pillar
+# attract
+# flamenco
+
+# Secondary requirements (i.e. pulled in from primary requirements)
+algoliasearch==1.8.0
+attrs==16.2.0
+bcrypt==2.0.0
+blinker==1.4
+bugsnag==2.3.1
+bleach==1.4.3
+Cerberus==0.9.2
+cffi==1.7.0
+commonmark==0.7.2
+cryptography==1.4
+enum34==1.1.6
+Eve==0.6.3
+Events==0.2.1
+Flask==0.10.1
+Flask-Cache==0.13.1
+Flask-Script==2.0.5
+Flask-Login==0.3.2
+Flask-OAuthlib==0.9.3
+Flask-PyMongo==0.4.1
+Flask-WTF==0.12
+flup==1.0.2
+future==0.15.2
+gcloud==0.12.0
+google-apitools==0.4.11
+googleapis-common-protos==1.2.0
+html5lib==0.9999999
+httplib2==0.9.2
+idna==2.0
+ipaddress==1.0.16
+itsdangerous==0.24
+Jinja2==2.8
+MarkupSafe==0.23
+markdown==2.6.7
+ndg-httpsclient==0.4.0
+oauth2client==3.0.0
+oauthlib==1.1.2
+pathlib2==2.2.1
+Pillow==2.8.1
+protobuf==3.0.0
+protorpc==0.11.1
+pyasn1==0.1.9
+pyasn1-modules==0.0.8
+pycparser==2.14
+pycrypto==2.6.1
+pylru==1.0.4
+pymongo==3.3.0
+pyOpenSSL==0.15.1
+python-dateutil==2.5.3
+redis==2.10.5
+requests==2.9.1
+requests-oauthlib==0.6.2
+rsa==3.4.2
+scandir==1.4
+simplejson==3.8.2
+six==1.10.0
+svn==0.3.43
+WebOb==1.5.0
+Werkzeug==0.11.10
+WTForms==2.1
+zencoder==0.6.5

rsync_ui.sh

@@ -1,14 +1,5 @@
 #!/usr/bin/env bash
 
-set -e  # error out when one of the commands in the script errors.
-
-if [ -z "$1" ]; then
-    echo "Usage: $0 {host-to-deploy-to}" >&2
-    exit 1
-fi
-
-DEPLOYHOST="$1"
-
 PILLAR_DIR=$(python <<EOT
 from __future__ import print_function
 import os.path
@@ -42,8 +33,8 @@ fi
 
 echo
 echo "*** SYNCING ASSETS ***"
-rsync -avh $ASSETS root@${DEPLOYHOST}:/data/git/pillar/pillar/web/static/assets/
+rsync -avh $ASSETS root@cloud.blender.org:/data/git/pillar/pillar/web/static/assets/
 
 echo
 echo "*** SYNCING TEMPLATES ***"
-rsync -avh $TEMPLATES root@${DEPLOYHOST}:/data/git/pillar/pillar/web/templates/
+rsync -avh $TEMPLATES root@cloud.blender.org:/data/git/pillar/pillar/web/templates/

runserver.wsgi

@@ -1,11 +1,14 @@
 from os.path import abspath, dirname
 import sys
 
+activate_this = '/data/venv/bin/activate_this.py'
+execfile(activate_this, dict(__file__=activate_this))
+from flup.server.fcgi import WSGIServer
 from pillar import PillarServer
 from attract import AttractExtension
 from flamenco import FlamencoExtension
 
-sys.path.append('/data/git/blender-cloud')
+sys.path.append('/data/git/blender-cloud/')
 
 attract = AttractExtension()
 flamenco = FlamencoExtension()
@@ -14,3 +17,6 @@ application = PillarServer(dirname(abspath(__file__)))
 application.load_extension(attract, '/attract')
 application.load_extension(flamenco, '/flamenco')
 application.process_extensions()
+
+if __name__ == '__main__':
+    WSGIServer(application).run()