Placing code + assets directly into Docker image

This radically changes the way we deploy to the production server, as a
Git checkout is no longer required there. All the necessary files are
now inside the docker image. As a result, /data/git should no longer be
mounted as a Docker volume.

- Renamed docker/build.sh → docker/full_rebuild.sh
  This makes it clearer that it performs a full rebuild of the Docker images.
- Full rebuilds should be done on a regular basis to pull in Ubuntu
  security updates.
- Removed rsync_ui.sh, we no longer need it. Other projects can also
  remove their rsync_ui.sh.
- Moved deploy.sh → deploy/2docker.sh and added deploy/2server.sh

.gitignore

@@ -18,3 +18,4 @@ node_modules/
 /google_app*.json
 /docker/2_buildpy/python/
 /docker/4_run/wheelhouse/
+/docker/4_run/deploy/

deploy.sh

@@ -1,164 +0,0 @@
-#!/bin/bash -e
-
-case $1 in
-    cloud*)
-        DEPLOYHOST="$1"
-        ;;
-    *)
-        echo "Use $0 cloud{nr}|cloud.blender.org" >&2
-        exit 1
-esac
-
-echo -n "Deploying to ${DEPLOYHOST}... "
-
-if ! ping ${DEPLOYHOST} -q -c 1 -W 2 >/dev/null; then
-    echo "host ${DEPLOYHOST} cannot be pinged, refusing to deploy." >&2
-    exit 2
-fi
-
-echo "press [ENTER] to continue, Ctrl+C to abort."
-read dummy
-
-
-# Deploys the current production branch to the production machine.
-PROJECT_NAME="blender-cloud"
-DOCKER_NAME="blender_cloud"
-CELERY_WORKER_DOCKER_NAME="celery_worker"
-CELERY_BEAT_DOCKER_NAME="celery_beat"
-REMOTE_ROOT="/data/git/${PROJECT_NAME}"
-
-SSH="ssh -o ClearAllForwardings=yes -o PermitLocalCommand=no ${DEPLOYHOST}"
-
-# macOS does not support readlink -f, so we use greadlink instead
-if [[ `uname` == 'Darwin' ]]; then
-    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
-    readlink='greadlink'
-else
-    readlink='readlink'
-fi
-
-ROOT="$(dirname "$($readlink -f "$0")")"
-cd ${ROOT}
-
-# Check that we're on production branch.
-if [ $(git rev-parse --abbrev-ref HEAD) != "production" ]; then
-    echo "You are NOT on the production branch, refusing to deploy." >&2
-    exit 1
-fi
-
-# Check that production branch has been pushed.
-if [ -n "$(git log origin/production..production --oneline)" ]; then
-    echo "WARNING: not all changes to the production branch have been pushed."
-    echo "Press [ENTER] to continue deploying current origin/production, CTRL+C to abort."
-    read dummy
-fi
-
-function find_module()
-{
-    MODULE_NAME=$1
-    MODULE_DIR=$(python <<EOT
-from __future__ import print_function
-import os.path
-try:
-    import ${MODULE_NAME}
-except ImportError:
-    raise SystemExit('${MODULE_NAME} not found on Python path. Are you in the correct venv?')
-
-print(os.path.dirname(os.path.dirname(${MODULE_NAME}.__file__)))
-EOT
-)
-
-    if [ $(git -C $MODULE_DIR rev-parse --abbrev-ref HEAD) != "production" ]; then
-        echo "${MODULE_NAME}: ($MODULE_DIR) NOT on the production branch, refusing to deploy." >&2
-        exit 1
-    fi
-
-    echo $MODULE_DIR
-}
-
-# Find our modules
-PILLAR_DIR=$(find_module pillar)
-ATTRACT_DIR=$(find_module attract)
-FLAMENCO_DIR=$(find_module flamenco)
-SVNMAN_DIR=$(find_module svnman)
-
-echo "Pillar  : $PILLAR_DIR"
-echo "Attract : $ATTRACT_DIR"
-echo "Flamenco: $FLAMENCO_DIR"
-echo "SVNMan  : $SVNMAN_DIR"
-
-if [ -z "$PILLAR_DIR" -o -z "$ATTRACT_DIR" -o -z "$FLAMENCO_DIR" -o -z "$SVNMAN_DIR" ];
-then
-    exit 1
-fi
-
-# SSH to cloud to pull all files in
-function git_pull() {
-    PROJECT_NAME="$1"
-    BRANCH="$2"
-    REMOTE_ROOT="/data/git/${PROJECT_NAME}"
-
-    echo "==================================================================="
-    echo "UPDATING FILES ON ${PROJECT_NAME}"
-    ${SSH} git -C ${REMOTE_ROOT} fetch origin ${BRANCH}
-    ${SSH} git -C ${REMOTE_ROOT} log origin/${BRANCH}..${BRANCH} --oneline
-    ${SSH} git -C ${REMOTE_ROOT} merge --ff-only origin/${BRANCH}
-}
-
-git_pull pillar-python-sdk master
-git_pull pillar production
-git_pull attract production
-git_pull flamenco production
-git_pull pillar-svnman production
-git_pull blender-cloud production
-
-# Update the virtualenv
-#${SSH} -t docker exec ${DOCKER_NAME} /data/venv/bin/pip install -U -r ${REMOTE_ROOT}/requirements.txt --exists-action w
-
-# RSync the world
-$ATTRACT_DIR/rsync_ui.sh ${DEPLOYHOST}
-$FLAMENCO_DIR/rsync_ui.sh ${DEPLOYHOST}
-$SVNMAN_DIR/rsync_ui.sh ${DEPLOYHOST}
-./rsync_ui.sh ${DEPLOYHOST}
-
-# Notify Sentry of this new deploy.
-# See https://sentry.io/blender-institute/blender-cloud/settings/release-tracking/
-# and https://docs.sentry.io/api/releases/post-organization-releases/
-# and https://sentry.io/api/
-echo
-echo "==================================================================="
-REVISION=$(date +'%Y%m%d.%H%M%S.%Z')
-echo "Notifying Sentry of this new deploy of revision ${REVISION}."
-SENTRY_RELEASE_URL="$(${SSH} python3 -c "\"import sys; sys.path.append('${REMOTE_ROOT}'); import config_local; print(config_local.SENTRY_RELEASE_URL)\"")"
-curl -vs "$SENTRY_RELEASE_URL" -XPOST -H 'Content-Type: application/json' -d "{\"version\": \"$REVISION\"}" | json_pp
-echo
-
-# Wait for [ENTER] to restart the server
-echo
-echo "==================================================================="
-echo "NOTE: If you want to edit config_local.py on the server, do so now."
-echo "NOTE: Press [ENTER] to continue and restart the server process."
-read dummy
-echo "Gracefully restarting server process"
-${SSH} docker exec ${DOCKER_NAME} apache2ctl graceful
-echo "Server process restarted"
-
-echo
-echo "==================================================================="
-echo "Restarting Celery worker."
-${SSH} docker restart ${CELERY_WORKER_DOCKER_NAME}
-echo "Celery worker docker restarted"
-echo "Restarting Celery beat."
-${SSH} docker restart ${CELERY_BEAT_DOCKER_NAME}
-echo "Celery beat docker restarted"
-
-
-echo
-echo "==================================================================="
-echo "Clearing front page from Redis cache."
-${SSH} docker exec redis redis-cli DEL pwview//
-
-echo
-echo "==================================================================="
-echo "Deploy of ${PROJECT_NAME} is done."
-echo "==================================================================="

deploy/2docker.sh

@@ -0,0 +1,124 @@
+#!/bin/bash -e
+
+DEPLOY_BRANCH=${DEPLOY_BRANCH:-production}
+
+# macOS does not support readlink -f, so we use greadlink instead
+if [[ `uname` == 'Darwin' ]]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
+
+ROOT="$(dirname "$(dirname "$($readlink -f "$0")")")"
+DEPLOYDIR="$ROOT/docker/4_run/deploy"
+PROJECT_NAME="$(basename $ROOT)"
+
+if [ -e $DEPLOYDIR ]; then
+    echo "$DEPLOYDIR already exists, press [ENTER] to DESTROY AND DEPLOY, Ctrl+C to abort."
+    read dummy
+    rm -rf $DEPLOYDIR
+else
+    echo -n "Deploying to $DEPLOYDIR… "
+    echo "press [ENTER] to continue, Ctrl+C to abort."
+    read dummy
+fi
+
+cd ${ROOT}
+mkdir -p $DEPLOYDIR
+REMOTE_ROOT="$DEPLOYDIR/$PROJECT_NAME"
+
+if [ -z "$SKIP_BRANCH_CHECK" ]; then
+    # Check that we're on production branch.
+    if [ $(git rev-parse --abbrev-ref HEAD) != "$DEPLOY_BRANCH" ]; then
+        echo "You are NOT on the $DEPLOY_BRANCH branch, refusing to deploy." >&2
+        exit 1
+    fi
+
+    # Check that production branch has been pushed.
+    if [ -n "$(git log origin/$DEPLOY_BRANCH..$DEPLOY_BRANCH --oneline)" ]; then
+        echo "WARNING: not all changes to the $DEPLOY_BRANCH branch have been pushed."
+        echo "Press [ENTER] to continue deploying current origin/$DEPLOY_BRANCH, CTRL+C to abort."
+        read dummy
+    fi
+fi
+
+function find_module()
+{
+    MODULE_NAME=$1
+    MODULE_DIR=$(python <<EOT
+from __future__ import print_function
+import os.path
+try:
+    import ${MODULE_NAME}
+except ImportError:
+    raise SystemExit('${MODULE_NAME} not found on Python path. Are you in the correct venv?')
+
+print(os.path.dirname(os.path.dirname(${MODULE_NAME}.__file__)))
+EOT
+)
+
+    if [ -z "$SKIP_BRANCH_CHECK" ]; then
+        if [ $(git -C $MODULE_DIR rev-parse --abbrev-ref HEAD) != "$DEPLOY_BRANCH" ]; then
+            echo "${MODULE_NAME}: ($MODULE_DIR) NOT on the $DEPLOY_BRANCH branch, refusing to deploy." >&2
+            exit 1
+        fi
+    fi
+
+    echo $MODULE_DIR
+}
+
+# Find our modules
+echo "==================================================================="
+echo "LOCAL MODULE LOCATIONS"
+PILLAR_DIR=$(find_module pillar)
+ATTRACT_DIR=$(find_module attract)
+FLAMENCO_DIR=$(find_module flamenco)
+SVNMAN_DIR=$(find_module svnman)
+SDK_DIR=$(find_module pillarsdk)
+
+echo "Pillar  : $PILLAR_DIR"
+echo "Attract : $ATTRACT_DIR"
+echo "Flamenco: $FLAMENCO_DIR"
+echo "SVNMan  : $SVNMAN_DIR"
+echo "SDK     : $SDK_DIR"
+
+if [ -z "$PILLAR_DIR" -o -z "$ATTRACT_DIR" -o -z "$FLAMENCO_DIR" -o -z "$SVNMAN_DIR" -o -z "$SDK_DIR" ];
+then
+    exit 1
+fi
+
+function git_clone() {
+    PROJECT_NAME="$1"
+    BRANCH="$2"
+    LOCAL_ROOT="$3"
+
+    echo "==================================================================="
+    echo "CLONING REPO ON $PROJECT_NAME @$BRANCH"
+    URL=$(git -C $LOCAL_ROOT remote get-url origin)
+    git -C $DEPLOYDIR clone --depth 1 --branch $BRANCH $URL $PROJECT_NAME
+}
+
+git_clone pillar-python-sdk master $SDK_DIR
+git_clone pillar $DEPLOY_BRANCH $PILLAR_DIR
+git_clone attract $DEPLOY_BRANCH $ATTRACT_DIR
+git_clone flamenco $DEPLOY_BRANCH $FLAMENCO_DIR
+git_clone pillar-svnman $DEPLOY_BRANCH $SVNMAN_DIR
+git_clone blender-cloud $DEPLOY_BRANCH $ROOT
+
+# Gulp everywhere
+GULP=$ROOT/node_modules/.bin/gulp
+if [ ! -e $GULP -o gulpfile.js -nt $GULP ]; then
+    npm install
+    touch $GULP  # installer doesn't always touch this after a build, so we do.
+fi
+$GULP --cwd $DEPLOYDIR/pillar --production
+$GULP --cwd $DEPLOYDIR/attract --production
+$GULP --cwd $DEPLOYDIR/flamenco --production
+$GULP --cwd $DEPLOYDIR/pillar-svnman --production
+$GULP --cwd $DEPLOYDIR/blender-cloud --production
+
+echo
+echo "==================================================================="
+echo "Deploy of ${PROJECT_NAME} is ready for dockerisation."
+echo "==================================================================="

deploy/2server.sh

@@ -0,0 +1,79 @@
+#!/bin/bash -e
+
+# macOS does not support readlink -f, so we use greadlink instead
+if [[ `uname` == 'Darwin' ]]; then
+    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
+    readlink='greadlink'
+else
+    readlink='readlink'
+fi
+ROOT="$(dirname "$(dirname "$($readlink -f "$0")")")"
+PROJECT_NAME="$(basename $ROOT)"
+DOCKER_DEPLOYDIR="$ROOT/docker/4_run/deploy"
+DOCKER_IMAGE="armadillica/blender_cloud:latest"
+REMOTE_SECRET_CONFIG_DIR="/data/config"
+REMOTE_DOCKER_COMPOSE_DIR="/root/docker"
+
+#################################################################################
+case $1 in
+    cloud*)
+        DEPLOYHOST="$1"
+        ;;
+    *)
+        echo "Use $0 cloud{nr}|cloud.blender.org" >&2
+        exit 1
+esac
+SSH_OPTS="-o ClearAllForwardings=yes -o PermitLocalCommand=no"
+SSH="ssh $SSH_OPTS $DEPLOYHOST"
+SCP="scp $SSH_OPTS"
+
+echo -n "Deploying to $DEPLOYHOST… "
+
+if ! ping $DEPLOYHOST -q -c 1 -W 2 >/dev/null; then
+    echo "host $DEPLOYHOST cannot be pinged, refusing to deploy." >&2
+    exit 2
+fi
+
+cat <<EOT
+[ping OK]
+
+Make sure that you have pushed the $DOCKER_IMAGE
+docker image to Docker Hub.
+
+press [ENTER] to continue, Ctrl+C to abort.
+EOT
+read dummy
+
+#################################################################################
+echo "==================================================================="
+echo "Bringing remote Docker up to date…"
+$SSH mkdir -p $REMOTE_DOCKER_COMPOSE_DIR
+$SCP $DOCKER_DEPLOYDIR/$PROJECT_NAME/docker/docker-compose.yml $DEPLOYHOST:$REMOTE_DOCKER_COMPOSE_DIR
+$SSH -T <<EOT
+set -e
+cd $REMOTE_DOCKER_COMPOSE_DIR
+docker pull $DOCKER_IMAGE
+docker-compose up -d
+
+echo
+echo "==================================================================="
+echo "Clearing front page from Redis cache."
+docker exec redis redis-cli DEL pwview//
+EOT
+
+# Notify Sentry of this new deploy.
+# See https://sentry.io/blender-institute/blender-cloud/settings/release-tracking/
+# and https://docs.sentry.io/api/releases/post-organization-releases/
+# and https://sentry.io/api/
+echo
+echo "==================================================================="
+REVISION=$(date +'%Y%m%d.%H%M%S.%Z')
+echo "Notifying Sentry of this new deploy of revision $REVISION."
+SENTRY_RELEASE_URL="$($SSH env PYTHONPATH="$REMOTE_SECRET_CONFIG_DIR" python3 -c "\"import config_secrets; print(config_secrets.SENTRY_RELEASE_URL)\"")"
+curl -s "$SENTRY_RELEASE_URL" -XPOST -H 'Content-Type: application/json' -d "{\"version\": \"$REVISION\"}" | json_pp
+echo
+
+echo
+echo "==================================================================="
+echo "Deploy to $DEPLOYHOST done."
+echo "==================================================================="

docker/4_run/config_local.py

@@ -0,0 +1,126 @@
+import os
+from collections import defaultdict
+DEBUG = False
+
+SCHEME = 'https'
+PREFERRED_URL_SCHEME = 'https'
+SERVER_NAME = 'cloud.blender.org'
+
+#os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = 'true'
+os.environ['PILLAR_MONGO_DBNAME'] = 'cloud'
+os.environ['PILLAR_MONGO_PORT'] = '27017'
+os.environ['PILLAR_MONGO_HOST'] = 'mongo'
+
+USE_X_SENDFILE = True
+
+STORAGE_BACKEND = 'gcs'
+
+CDN_SERVICE_DOMAIN = 'blendercloud-pro.r.worldssl.net'
+CDN_CONTENT_SUBFOLDER = ''
+CDN_STORAGE_ADDRESS = 'push-11.cdnsun.com'
+
+CACHE_TYPE = 'redis' #null
+CACHE_KEY_PREFIX = 'pw_'
+CACHE_REDIS_HOST = 'redis'
+CACHE_REDIS_PORT = '6379'
+CACHE_REDIS_URL = 'redis://redis:6379'
+
+PILLAR_SERVER_ENDPOINT = 'https://cloud.blender.org/api/'
+
+BLENDER_ID_ENDPOINT = 'https://www.blender.org/id/'
+
+GCLOUD_APP_CREDENTIALS = '/data/config/google_app.json'
+GCLOUD_PROJECT = 'blender-cloud'
+
+MAIN_PROJECT_ID = '563a9c8cf0e722006ce97b03'
+# MAIN_PROJECT_ID = '57aa07c088bef606e89078bd'
+
+ALGOLIA_INDEX_USERS = 'pro_Users'
+ALGOLIA_INDEX_NODES = 'pro_Nodes'
+
+ZENCODER_NOTIFICATIONS_URL = 'https://cloud.blender.org/api/encoding/zencoder/notifications'
+
+FILE_LINK_VALIDITY = defaultdict(
+    lambda: 3600 * 24 * 30,  # default of 1 month.
+    gcs=3600 * 23,  # 23 hours for Google Cloud Storage.
+    cdnsun=3600 * 23
+)
+
+LOGGING = {
+    'version': 1,
+    'formatters': {
+        'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'default',
+            'stream': 'ext://sys.stderr',
+        }
+    },
+    'loggers': {
+        'pillar': {'level': 'INFO'},
+        #'pillar.auth': {'level': 'DEBUG'},
+        #'pillar.api.blender_id': {'level': 'DEBUG'},
+        #'pillar.api.blender_cloud.subscription': {'level': 'DEBUG'},
+        'bcloud': {'level': 'INFO'},
+        'cloud': {'level': 'INFO'},
+        'attract': {'level': 'INFO'},
+        'flamenco': {'level': 'INFO'},
+        #'pillar.api.file_storage': {'level': 'DEBUG'},
+        #'pillar.api.file_storage.ensure_valid_link': {'level': 'INFO'},
+        'pillar.api.file_storage.refresh_links_for_backend': {'level': 'DEBUG'},
+        'werkzeug': {'level': 'DEBUG'},
+        'eve': {'level': 'WARNING'},
+        #'elasticsearch': {'level': 'DEBUG'},
+    },
+    'root': {
+        'level': 'WARNING',
+        'handlers': [
+            'console',
+        ],
+    }
+}
+
+REDIRECTS = {
+    # For old links, refer to the services page (hopefully it refreshes then)
+    'downloads/blender_cloud-latest-bundle.zip': 'https://cloud.blender.org/services#blender-addon',
+
+    # Latest Blender Cloud add-on; remember to update BLENDER_CLOUD_ADDON_VERSION.
+    'downloads/blender_cloud-latest-addon.zip':
+        'https://storage.googleapis.com/institute-storage/addons/blender_cloud-1.8.0.addon.zip',
+
+    # Redirect old Grafista endpoint to /stats
+    '/stats/': '/stats',
+}
+
+# Latest version of the add-on; remember to update REDIRECTS.
+BLENDER_CLOUD_ADDON_VERSION = '1.8.0'
+
+UTM_LINKS = {
+    'cartoon_brew': {
+        'image': 'https://imgur.com/13nQTi3.png',
+        'link': 'https://store.blender.org/product/membership/'
+    }
+}
+
+# Disabled until we have regenerated the majority of the links.
+CELERY_BEAT_SCHEDULE = {
+    'regenerate-expired-links': {
+        'task': 'pillar.celery.file_link_tasks.regenerate_all_expired_links',
+        'schedule': 600,  # every N seconds
+        'args': ('gcs', 500)
+    },
+}
+
+SVNMAN_REPO_URL = 'https://svn.blender.cloud/repo/'
+SVNMAN_API_URL = 'https://svn.blender.cloud/api/'
+
+# Mail options, see pillar.celery.email_tasks.
+SMTP_HOST = 'proog.blender.org'
+SMTP_PORT = 25
+SMTP_USERNAME = 'server@blender.cloud'
+SMTP_TIMEOUT = 30  # timeout in seconds, https://docs.python.org/3/library/smtplib.html#smtplib.SMTP
+MAIL_RETRY = 180  # in seconds, delay until trying to send an email again.
+MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
+MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@blender.org'

docker/4_run/run.docker

@@ -27,7 +27,9 @@ RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR
 ADD wheelhouse /data/wheelhouse
 RUN pip3 install --no-index --find-links=/data/wheelhouse -r /data/wheelhouse/requirements.txt
 
-VOLUME /data/git
+ADD deploy /data/git
+ADD config_local.py /data/git/blender-cloud/
+
 VOLUME /data/config
 VOLUME /data/storage
 VOLUME /var/log

docker/docker-compose.yml

@@ -9,6 +9,11 @@ services:
       - /data/storage/db-bak:/data/db-bak # for backing up stuff etc.
     ports:
       - "127.0.0.1:27017:27017"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   redis:
     image:                   redis:3.2.8
@@ -16,6 +21,11 @@ services:
     restart:                 always
     ports:
       - "127.0.0.1:6379:6379"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   rabbit:
     image:                   rabbitmq:3.6.10
@@ -23,6 +33,11 @@ services:
     restart:                 always
     ports:
       - "127.0.0.1:5672:5672"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   elastic:
     image:                   armadillica/elasticsearch:6.1.1
@@ -35,6 +50,11 @@ services:
       - "127.0.0.1:9200:9200"
     environment:
       ES_JAVA_OPTS:          "-Xms256m -Xmx256m"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   elasticproxy:
     image:                   armadillica/elasticproxy:1.2
@@ -43,6 +63,11 @@ services:
     command:                 /elasticproxy -elastic http://elastic:9200/
     depends_on:
       - elastic
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   kibana:
     image:                   armadillica/kibana:6.1.1
@@ -60,6 +85,11 @@ services:
       NODE_OPTIONS:          "--max-old-space-size=200"
     depends_on:
       - elasticproxy
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   blender_cloud:
     image:                   armadillica/blender_cloud:latest
@@ -70,9 +100,9 @@ services:
       VIRTUAL_HOST_WEIGHT:   10
       FORCE_SSL:             "true"
       GZIP_COMPRESSION_TYPE: "text/html text/plain text/css application/javascript"
+      PILLAR_CONFIG:         /data/config/config_secrets.py
     volumes:
       # format:              HOST:CONTAINER
-      - /data/git:/data/git:ro
       - /data/config:/data/config:ro
       - /data/storage/pillar:/data/storage/pillar
       - /data/log:/var/log
@@ -86,9 +116,10 @@ services:
     entrypoint:              /celery-worker.sh
     container_name:          celery_worker
     restart:                 always
+    environment:
+      PILLAR_CONFIG:         /data/config/config_secrets.py
     volumes:
       # format:              HOST:CONTAINER
-      - /data/git:/data/git:ro
       - /data/config:/data/config:ro
       - /data/storage/pillar:/data/storage/pillar
       - /data/log:/var/log
@@ -96,34 +127,34 @@ services:
       - mongo
       - redis
       - rabbit
-
-  celery_beat:
-    image:                   armadillica/blender_cloud:latest
-    entrypoint:              /celery-beat.sh
-    container_name:          celery_beat
-    restart:                 always
-    volumes:
-      # format:              HOST:CONTAINER
-      - /data/git:/data/git:ro
-      - /data/storage/pillar:/data/storage/pillar
-      - /data/log:/var/log
-    depends_on:
-      - mongo
-      - redis
-      - rabbit
     logging:
       driver:                "json-file"
       options:
         max-size:            "200k"
         max-file:            "20"
 
-  grafista:
+  celery_beat:
-    image:                   armadillica/grafista:latest
+    image:                   armadillica/blender_cloud:latest
-    container_name:          grafista
+    entrypoint:              /celery-beat.sh
+    container_name:          celery_beat
     restart:                 always
+    environment:
+      PILLAR_CONFIG:         /data/config/config_secrets.py
     volumes:
-      - /data/git/grafista:/data/git/grafista:ro
+      # format:              HOST:CONTAINER
-      - /data/storage/grafista:/data/storage/grafista
+      - /data/config:/data/config:ro
+      - /data/storage/pillar:/data/storage/pillar
+      - /data/log:/var/log
+    depends_on:
+      - mongo
+      - redis
+      - rabbit
+      - celery_worker
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"
 
   letsencrypt:
     image:                   armadillica/picohttp:latest

package.json

@@ -15,6 +15,7 @@
 		"gulp-if": "^2.0.1",
 		"gulp-git": "~2.4.2",
 		"gulp-pug": "~3.2.0",
+		"gulp-jade": "~1.1.0",
 		"gulp-livereload": "~3.8.1",
 		"gulp-plumber": "~1.1.0",
 		"gulp-rename": "~1.2.2",

rsync_ui.sh

@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-
-set -e  # error out when one of the commands in the script errors.
-
-if [ -z "$1" ]; then
-    echo "Usage: $0 {host-to-deploy-to}" >&2
-    exit 1
-fi
-
-DEPLOYHOST="$1"
-
-# macOS does not support readlink -f, so we use greadlink instead
-if [[ `uname` == 'Darwin' ]]; then
-    command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; }
-    readlink='greadlink'
-else
-    readlink='readlink'
-fi
-
-BLENDER_CLOUD_DIR="$(dirname "$($readlink -f "$0")")"
-if [ ! -d "$BLENDER_CLOUD_DIR" ]; then
-    echo "Unable to find Blender Cloud dir '$BLENDER_CLOUD_DIR'"
-    exit 1
-fi
-
-BLENDER_CLOUD_ASSETS="$BLENDER_CLOUD_DIR/cloud/static/"
-BLENDER_CLOUD_TEMPLATES="$BLENDER_CLOUD_DIR/cloud/templates/"
-
-if [ ! -d "$BLENDER_CLOUD_ASSETS" ]; then
-    echo "Unable to find assets dir $BLENDER_CLOUD_ASSETS"
-    exit 1
-fi
-
-cd $BLENDER_CLOUD_DIR
-if [ $(git rev-parse --abbrev-ref HEAD) != "production" ]; then
-    echo "You are NOT on the production branch, refusing to rsync_ui." >&2
-    exit 1
-fi
-
-PILLAR_DIR=$(python <<EOT
-from __future__ import print_function
-import os.path
-import pillar
-
-print(os.path.dirname(os.path.dirname(pillar.__file__)))
-EOT
-)
-
-PILLAR_ASSETS="$PILLAR_DIR/pillar/web/static/assets/"
-PILLAR_TEMPLATES="$PILLAR_DIR/pillar/web/templates/"
-
-if [ ! -d "$PILLAR_ASSETS" ]; then
-    echo "Unable to find assets dir $PILLAR_ASSETS"
-    exit 1
-fi
-
-cd $PILLAR_DIR
-if [ $(git rev-parse --abbrev-ref HEAD) != "production" ]; then
-    echo "Pillar is NOT on the production branch, refusing to rsync_ui." >&2
-    exit 1
-fi
-
-echo
-echo "*** GULPA GULPA PILLAR ***"
-# TODO(Pablo): this command fails when passing the --production CLI
-# arg.
-./gulp
-
-echo
-echo "*** SYNCING PILLAR_ASSETS ***"
-rsync -avh $PILLAR_ASSETS root@${DEPLOYHOST}:/data/git/pillar/pillar/web/static/assets/ --delete-after
-
-echo
-echo "*** SYNCING PILLAR_TEMPLATES ***"
-rsync -avh $PILLAR_TEMPLATES root@${DEPLOYHOST}:/data/git/pillar/pillar/web/templates/ --delete-after
-
-
-cd $BLENDER_CLOUD_DIR
-
-echo
-echo "*** GULPA GULPA BLENDER_CLOUD ***"
-./gulp --production
-
-echo
-echo "*** SYNCING BLENDER_CLOUD_ASSETS ***"
-# Exclude files managed by Git.
-rsync -avh $BLENDER_CLOUD_ASSETS --exclude js/vendor/ root@${DEPLOYHOST}:/data/git/blender-cloud/cloud/static/ --delete-after
-
-echo
-echo "*** SYNCING BLENDER_CLOUD_TEMPLATES ***"
-rsync -avh $BLENDER_CLOUD_TEMPLATES root@${DEPLOYHOST}:/data/git/blender-cloud/cloud/templates/ --delete-after