diff --git a/.gitignore b/.gitignore index ae97b58..21e68b9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ node_modules/ /google_app*.json /docker/2_buildpy/python/ /docker/4_run/wheelhouse/ +/docker/4_run/deploy/ diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 1705622..0000000 --- a/deploy.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash -e - -case $1 in - cloud*) - DEPLOYHOST="$1" - ;; - *) - echo "Use $0 cloud{nr}|cloud.blender.org" >&2 - exit 1 -esac - -echo -n "Deploying to ${DEPLOYHOST}... " - -if ! ping ${DEPLOYHOST} -q -c 1 -W 2 >/dev/null; then - echo "host ${DEPLOYHOST} cannot be pinged, refusing to deploy." >&2 - exit 2 -fi - -echo "press [ENTER] to continue, Ctrl+C to abort." -read dummy - - -# Deploys the current production branch to the production machine. -PROJECT_NAME="blender-cloud" -DOCKER_NAME="blender_cloud" -CELERY_WORKER_DOCKER_NAME="celery_worker" -CELERY_BEAT_DOCKER_NAME="celery_beat" -REMOTE_ROOT="/data/git/${PROJECT_NAME}" - -SSH="ssh -o ClearAllForwardings=yes -o PermitLocalCommand=no ${DEPLOYHOST}" - -# macOS does not support readlink -f, so we use greadlink instead -if [[ `uname` == 'Darwin' ]]; then - command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } - readlink='greadlink' -else - readlink='readlink' -fi - -ROOT="$(dirname "$($readlink -f "$0")")" -cd ${ROOT} - -# Check that we're on production branch. -if [ $(git rev-parse --abbrev-ref HEAD) != "production" ]; then - echo "You are NOT on the production branch, refusing to deploy." >&2 - exit 1 -fi - -# Check that production branch has been pushed. -if [ -n "$(git log origin/production..production --oneline)" ]; then - echo "WARNING: not all changes to the production branch have been pushed." - echo "Press [ENTER] to continue deploying current origin/production, CTRL+C to abort." - read dummy -fi - -function find_module() -{ - MODULE_NAME=$1 - MODULE_DIR=$(python <&2 - exit 1 - fi - - echo $MODULE_DIR -} - -# Find our modules -PILLAR_DIR=$(find_module pillar) -ATTRACT_DIR=$(find_module attract) -FLAMENCO_DIR=$(find_module flamenco) -SVNMAN_DIR=$(find_module svnman) - -echo "Pillar : $PILLAR_DIR" -echo "Attract : $ATTRACT_DIR" -echo "Flamenco: $FLAMENCO_DIR" -echo "SVNMan : $SVNMAN_DIR" - -if [ -z "$PILLAR_DIR" -o -z "$ATTRACT_DIR" -o -z "$FLAMENCO_DIR" -o -z "$SVNMAN_DIR" ]; -then - exit 1 -fi - -# SSH to cloud to pull all files in -function git_pull() { - PROJECT_NAME="$1" - BRANCH="$2" - REMOTE_ROOT="/data/git/${PROJECT_NAME}" - - echo "===================================================================" - echo "UPDATING FILES ON ${PROJECT_NAME}" - ${SSH} git -C ${REMOTE_ROOT} fetch origin ${BRANCH} - ${SSH} git -C ${REMOTE_ROOT} log origin/${BRANCH}..${BRANCH} --oneline - ${SSH} git -C ${REMOTE_ROOT} merge --ff-only origin/${BRANCH} -} - -git_pull pillar-python-sdk master -git_pull pillar production -git_pull attract production -git_pull flamenco production -git_pull pillar-svnman production -git_pull blender-cloud production - -# Update the virtualenv -#${SSH} -t docker exec ${DOCKER_NAME} /data/venv/bin/pip install -U -r ${REMOTE_ROOT}/requirements.txt --exists-action w - -# RSync the world -$ATTRACT_DIR/rsync_ui.sh ${DEPLOYHOST} -$FLAMENCO_DIR/rsync_ui.sh ${DEPLOYHOST} -$SVNMAN_DIR/rsync_ui.sh ${DEPLOYHOST} -./rsync_ui.sh ${DEPLOYHOST} - -# Notify Sentry of this new deploy. -# See https://sentry.io/blender-institute/blender-cloud/settings/release-tracking/ -# and https://docs.sentry.io/api/releases/post-organization-releases/ -# and https://sentry.io/api/ -echo -echo "===================================================================" -REVISION=$(date +'%Y%m%d.%H%M%S.%Z') -echo "Notifying Sentry of this new deploy of revision ${REVISION}." -SENTRY_RELEASE_URL="$(${SSH} python3 -c "\"import sys; sys.path.append('${REMOTE_ROOT}'); import config_local; print(config_local.SENTRY_RELEASE_URL)\"")" -curl -vs "$SENTRY_RELEASE_URL" -XPOST -H 'Content-Type: application/json' -d "{\"version\": \"$REVISION\"}" | json_pp -echo - -# Wait for [ENTER] to restart the server -echo -echo "===================================================================" -echo "NOTE: If you want to edit config_local.py on the server, do so now." -echo "NOTE: Press [ENTER] to continue and restart the server process." -read dummy -echo "Gracefully restarting server process" -${SSH} docker exec ${DOCKER_NAME} apache2ctl graceful -echo "Server process restarted" - -echo -echo "===================================================================" -echo "Restarting Celery worker." -${SSH} docker restart ${CELERY_WORKER_DOCKER_NAME} -echo "Celery worker docker restarted" -echo "Restarting Celery beat." -${SSH} docker restart ${CELERY_BEAT_DOCKER_NAME} -echo "Celery beat docker restarted" - - -echo -echo "===================================================================" -echo "Clearing front page from Redis cache." -${SSH} docker exec redis redis-cli DEL pwview// - -echo -echo "===================================================================" -echo "Deploy of ${PROJECT_NAME} is done." -echo "===================================================================" diff --git a/deploy/2docker.sh b/deploy/2docker.sh new file mode 100755 index 0000000..a43382e --- /dev/null +++ b/deploy/2docker.sh @@ -0,0 +1,124 @@ +#!/bin/bash -e + +DEPLOY_BRANCH=${DEPLOY_BRANCH:-production} + +# macOS does not support readlink -f, so we use greadlink instead +if [[ `uname` == 'Darwin' ]]; then + command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } + readlink='greadlink' +else + readlink='readlink' +fi + +ROOT="$(dirname "$(dirname "$($readlink -f "$0")")")" +DEPLOYDIR="$ROOT/docker/4_run/deploy" +PROJECT_NAME="$(basename $ROOT)" + +if [ -e $DEPLOYDIR ]; then + echo "$DEPLOYDIR already exists, press [ENTER] to DESTROY AND DEPLOY, Ctrl+C to abort." + read dummy + rm -rf $DEPLOYDIR +else + echo -n "Deploying to $DEPLOYDIR… " + echo "press [ENTER] to continue, Ctrl+C to abort." + read dummy +fi + +cd ${ROOT} +mkdir -p $DEPLOYDIR +REMOTE_ROOT="$DEPLOYDIR/$PROJECT_NAME" + +if [ -z "$SKIP_BRANCH_CHECK" ]; then + # Check that we're on production branch. + if [ $(git rev-parse --abbrev-ref HEAD) != "$DEPLOY_BRANCH" ]; then + echo "You are NOT on the $DEPLOY_BRANCH branch, refusing to deploy." >&2 + exit 1 + fi + + # Check that production branch has been pushed. + if [ -n "$(git log origin/$DEPLOY_BRANCH..$DEPLOY_BRANCH --oneline)" ]; then + echo "WARNING: not all changes to the $DEPLOY_BRANCH branch have been pushed." + echo "Press [ENTER] to continue deploying current origin/$DEPLOY_BRANCH, CTRL+C to abort." + read dummy + fi +fi + +function find_module() +{ + MODULE_NAME=$1 + MODULE_DIR=$(python <&2 + exit 1 + fi + fi + + echo $MODULE_DIR +} + +# Find our modules +echo "===================================================================" +echo "LOCAL MODULE LOCATIONS" +PILLAR_DIR=$(find_module pillar) +ATTRACT_DIR=$(find_module attract) +FLAMENCO_DIR=$(find_module flamenco) +SVNMAN_DIR=$(find_module svnman) +SDK_DIR=$(find_module pillarsdk) + +echo "Pillar : $PILLAR_DIR" +echo "Attract : $ATTRACT_DIR" +echo "Flamenco: $FLAMENCO_DIR" +echo "SVNMan : $SVNMAN_DIR" +echo "SDK : $SDK_DIR" + +if [ -z "$PILLAR_DIR" -o -z "$ATTRACT_DIR" -o -z "$FLAMENCO_DIR" -o -z "$SVNMAN_DIR" -o -z "$SDK_DIR" ]; +then + exit 1 +fi + +function git_clone() { + PROJECT_NAME="$1" + BRANCH="$2" + LOCAL_ROOT="$3" + + echo "===================================================================" + echo "CLONING REPO ON $PROJECT_NAME @$BRANCH" + URL=$(git -C $LOCAL_ROOT remote get-url origin) + git -C $DEPLOYDIR clone --depth 1 --branch $BRANCH $URL $PROJECT_NAME +} + +git_clone pillar-python-sdk master $SDK_DIR +git_clone pillar $DEPLOY_BRANCH $PILLAR_DIR +git_clone attract $DEPLOY_BRANCH $ATTRACT_DIR +git_clone flamenco $DEPLOY_BRANCH $FLAMENCO_DIR +git_clone pillar-svnman $DEPLOY_BRANCH $SVNMAN_DIR +git_clone blender-cloud $DEPLOY_BRANCH $ROOT + +# Gulp everywhere +GULP=$ROOT/node_modules/.bin/gulp +if [ ! -e $GULP -o gulpfile.js -nt $GULP ]; then + npm install + touch $GULP # installer doesn't always touch this after a build, so we do. +fi +$GULP --cwd $DEPLOYDIR/pillar --production +$GULP --cwd $DEPLOYDIR/attract --production +$GULP --cwd $DEPLOYDIR/flamenco --production +$GULP --cwd $DEPLOYDIR/pillar-svnman --production +$GULP --cwd $DEPLOYDIR/blender-cloud --production + +echo +echo "===================================================================" +echo "Deploy of ${PROJECT_NAME} is ready for dockerisation." +echo "===================================================================" diff --git a/deploy/2server.sh b/deploy/2server.sh new file mode 100755 index 0000000..3fa04b4 --- /dev/null +++ b/deploy/2server.sh @@ -0,0 +1,79 @@ +#!/bin/bash -e + +# macOS does not support readlink -f, so we use greadlink instead +if [[ `uname` == 'Darwin' ]]; then + command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } + readlink='greadlink' +else + readlink='readlink' +fi +ROOT="$(dirname "$(dirname "$($readlink -f "$0")")")" +PROJECT_NAME="$(basename $ROOT)" +DOCKER_DEPLOYDIR="$ROOT/docker/4_run/deploy" +DOCKER_IMAGE="armadillica/blender_cloud:latest" +REMOTE_SECRET_CONFIG_DIR="/data/config" +REMOTE_DOCKER_COMPOSE_DIR="/root/docker" + +################################################################################# +case $1 in + cloud*) + DEPLOYHOST="$1" + ;; + *) + echo "Use $0 cloud{nr}|cloud.blender.org" >&2 + exit 1 +esac +SSH_OPTS="-o ClearAllForwardings=yes -o PermitLocalCommand=no" +SSH="ssh $SSH_OPTS $DEPLOYHOST" +SCP="scp $SSH_OPTS" + +echo -n "Deploying to $DEPLOYHOST… " + +if ! ping $DEPLOYHOST -q -c 1 -W 2 >/dev/null; then + echo "host $DEPLOYHOST cannot be pinged, refusing to deploy." >&2 + exit 2 +fi + +cat <&2 - exit 1 -fi - -DEPLOYHOST="$1" - -# macOS does not support readlink -f, so we use greadlink instead -if [[ `uname` == 'Darwin' ]]; then - command -v greadlink 2>/dev/null 2>&1 || { echo >&2 "Install greadlink using brew."; exit 1; } - readlink='greadlink' -else - readlink='readlink' -fi - -BLENDER_CLOUD_DIR="$(dirname "$($readlink -f "$0")")" -if [ ! -d "$BLENDER_CLOUD_DIR" ]; then - echo "Unable to find Blender Cloud dir '$BLENDER_CLOUD_DIR'" - exit 1 -fi - -BLENDER_CLOUD_ASSETS="$BLENDER_CLOUD_DIR/cloud/static/" -BLENDER_CLOUD_TEMPLATES="$BLENDER_CLOUD_DIR/cloud/templates/" - -if [ ! -d "$BLENDER_CLOUD_ASSETS" ]; then - echo "Unable to find assets dir $BLENDER_CLOUD_ASSETS" - exit 1 -fi - -cd $BLENDER_CLOUD_DIR -if [ $(git rev-parse --abbrev-ref HEAD) != "production" ]; then - echo "You are NOT on the production branch, refusing to rsync_ui." >&2 - exit 1 -fi - -PILLAR_DIR=$(python <&2 - exit 1 -fi - -echo -echo "*** GULPA GULPA PILLAR ***" -# TODO(Pablo): this command fails when passing the --production CLI -# arg. -./gulp - -echo -echo "*** SYNCING PILLAR_ASSETS ***" -rsync -avh $PILLAR_ASSETS root@${DEPLOYHOST}:/data/git/pillar/pillar/web/static/assets/ --delete-after - -echo -echo "*** SYNCING PILLAR_TEMPLATES ***" -rsync -avh $PILLAR_TEMPLATES root@${DEPLOYHOST}:/data/git/pillar/pillar/web/templates/ --delete-after - - -cd $BLENDER_CLOUD_DIR - -echo -echo "*** GULPA GULPA BLENDER_CLOUD ***" -./gulp --production - -echo -echo "*** SYNCING BLENDER_CLOUD_ASSETS ***" -# Exclude files managed by Git. -rsync -avh $BLENDER_CLOUD_ASSETS --exclude js/vendor/ root@${DEPLOYHOST}:/data/git/blender-cloud/cloud/static/ --delete-after - -echo -echo "*** SYNCING BLENDER_CLOUD_TEMPLATES ***" -rsync -avh $BLENDER_CLOUD_TEMPLATES root@${DEPLOYHOST}:/data/git/blender-cloud/cloud/templates/ --delete-after