From 881cb6bc40763776c2d5118168229b75ac5d2076 Mon Sep 17 00:00:00 2001 From: Francesco Siddi Date: Wed, 10 Aug 2016 18:27:27 +0200 Subject: [PATCH] Staged Docker build script --- .gitignore | 3 ++ cloud-manage.py | 7 --- docker/1_base/base.docker | 16 ++++++ docker/1_base/build.sh | 3 ++ docker/2_build/build-wheels.sh | 3 ++ docker/2_build/build.docker | 26 +++++++++ docker/2_build/build.sh | 11 ++++ docker/3_run/000-default.conf | 30 +++++++++++ docker/3_run/build.sh | 5 ++ docker/3_run/docker-entrypoint.sh | 19 +++++++ docker/3_run/manage.sh | 3 ++ docker/3_run/run.docker | 45 ++++++++++++++++ docker/build.sh | 16 ++++++ manage.py | 90 ++----------------------------- requirements.txt | 31 +++++++++++ runserver.wsgi | 4 +- 16 files changed, 217 insertions(+), 95 deletions(-) delete mode 100755 cloud-manage.py create mode 100755 docker/1_base/base.docker create mode 100644 docker/1_base/build.sh create mode 100644 docker/2_build/build-wheels.sh create mode 100755 docker/2_build/build.docker create mode 100755 docker/2_build/build.sh create mode 100644 docker/3_run/000-default.conf create mode 100755 docker/3_run/build.sh create mode 100644 docker/3_run/docker-entrypoint.sh create mode 100644 docker/3_run/manage.sh create mode 100755 docker/3_run/run.docker create mode 100755 docker/build.sh create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 840b937..f1dae4e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ config_local.py /.eggs/ /dump/ /google_app*.json + +docker/3_run/wheelhouse/ +static/assets/img/ diff --git a/cloud-manage.py b/cloud-manage.py deleted file mode 100755 index 02324ad..0000000 --- a/cloud-manage.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -from pillar_server import cli -from cloud import app - -cli.manager.app = app -cli.manager.run() diff --git a/docker/1_base/base.docker b/docker/1_base/base.docker new file mode 100755 index 0000000..3718085 --- /dev/null +++ b/docker/1_base/base.docker @@ -0,0 +1,16 @@ +FROM ubuntu:16.04 +MAINTAINER Francesco Siddi + +RUN apt-get update && apt-get install -qyy \ + -o APT::Install-Recommends=false -o APT::Install-Suggests=false \ + python-pip libffi6 openssl ffmpeg + +RUN mkdir -p /data/git/pillar \ +&& mkdir -p /data/storage \ +&& mkdir -p /data/config \ +&& mkdir -p /data/venv \ +&& mkdir -p /data/wheelhouse + +RUN pip install virtualenv +RUN virtualenv /data/venv +RUN . /data/venv/bin/activate && pip install -U pip && pip install wheel diff --git a/docker/1_base/build.sh b/docker/1_base/build.sh new file mode 100644 index 0000000..40aa7c2 --- /dev/null +++ b/docker/1_base/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t pillar_base -f base.docker .; diff --git a/docker/2_build/build-wheels.sh b/docker/2_build/build-wheels.sh new file mode 100644 index 0000000..56cc2bc --- /dev/null +++ b/docker/2_build/build-wheels.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +. /data/venv/bin/activate && pip wheel --wheel-dir=/data/wheelhouse -r /requirements.txt diff --git a/docker/2_build/build.docker b/docker/2_build/build.docker new file mode 100755 index 0000000..0a5668b --- /dev/null +++ b/docker/2_build/build.docker @@ -0,0 +1,26 @@ +FROM pillar_base +MAINTAINER Francesco Siddi + +RUN apt-get update && apt-get install -qy \ +git \ +gcc \ +libffi-dev \ +libssl-dev \ +pypy-dev \ +python-dev \ +python-imaging \ +zlib1g-dev \ +libjpeg-dev \ +libtiff-dev \ +python-crypto \ +python-openssl + +ENV WHEELHOUSE=/data/wheelhouse +ENV PIP_WHEEL_DIR=/data/wheelhouse +ENV PIP_FIND_LINKS=/data/wheelhouse + +VOLUME /data/wheelhouse + +ADD requirements.txt /requirements.txt +ADD build-wheels.sh /build-wheels.sh +ENTRYPOINT ["bash", "build-wheels.sh"] diff --git a/docker/2_build/build.sh b/docker/2_build/build.sh new file mode 100755 index 0000000..533b149 --- /dev/null +++ b/docker/2_build/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +mkdir -p ../3_run/wheelhouse; +cp ../../requirements.txt .; + +docker build -t pillar_build -f build.docker .; +docker run --rm \ + -v "$(pwd)"/../3_run/wheelhouse:/data/wheelhouse \ + pillar_build; + +rm requirements.txt; diff --git a/docker/3_run/000-default.conf b/docker/3_run/000-default.conf new file mode 100644 index 0000000..80a9f17 --- /dev/null +++ b/docker/3_run/000-default.conf @@ -0,0 +1,30 @@ + + # EnableSendfile on + XSendFile on + XSendFilePath /data/storage/cloud + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + # Available loglevels: trace8, ..., trace1, debug, info, notice, warn, + # error, crit, alert, emerg. + # It is also possible to configure the loglevel for particular + # modules, e.g. + # LogLevel info ssl:warn + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + WSGIDaemonProcess cloud + WSGIPassAuthorization On + + WSGIScriptAlias / /data/git/blender-cloud/runserver.wsgi \ + process-group=cloud application-group=%{GLOBAL} + + + + Require all granted + + + + diff --git a/docker/3_run/build.sh b/docker/3_run/build.sh new file mode 100755 index 0000000..5853b58 --- /dev/null +++ b/docker/3_run/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +cp ../../requirements.txt .; +docker build -t armadillica/blender_cloud -f run.docker .; +rm requirements.txt; diff --git a/docker/3_run/docker-entrypoint.sh b/docker/3_run/docker-entrypoint.sh new file mode 100644 index 0000000..bf022f1 --- /dev/null +++ b/docker/3_run/docker-entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [ ! -f /installed ]; then + echo "Installing pillar and pillarskd" + # . /data/venv/bin/activate && pip install -e /data/git/pillar + ln -s /data/git/pillar/pillar /data/venv/lib/python2.7/site-packages/pillar + # . /data/venv/bin/activate && pip install -e /data/git/pillar-python-sdk + ln -s /data/git/pillar-python-sdk/pillarsdk /data/venv/lib/python2.7/site-packages/pillarsdk + touch installed +fi + +if [ "$DEV" = "true" ]; then + echo "Running in development mode" + bash manage.sh runserver +else + # Run Apache + a2enmod rewrite + /usr/sbin/apache2ctl -D FOREGROUND +fi diff --git a/docker/3_run/manage.sh b/docker/3_run/manage.sh new file mode 100644 index 0000000..e1f5e9e --- /dev/null +++ b/docker/3_run/manage.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +. /data/venv/bin/activate && python /data/git/blender-cloud/manage.py $@ diff --git a/docker/3_run/run.docker b/docker/3_run/run.docker new file mode 100755 index 0000000..36d2e3c --- /dev/null +++ b/docker/3_run/run.docker @@ -0,0 +1,45 @@ +FROM pillar_base + +RUN apt-get update && apt-get install -qyy \ +-o APT::Install-Recommends=true -o APT::Install-Suggests=false \ +git \ +apache2 \ +libapache2-mod-wsgi \ +libapache2-mod-xsendfile \ +libjpeg8 \ +libtiff5 \ +&& rm -rf /var/lib/apt/lists/* + +ENV APACHE_RUN_USER www-data +ENV APACHE_RUN_GROUP www-data +ENV APACHE_LOG_DIR /var/log/apache2 +ENV APACHE_PID_FILE /var/run/apache2.pid +ENV APACHE_RUN_DIR /var/run/apache2 +ENV APACHE_LOCK_DIR /var/lock/apache2 + +RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR + +ADD requirements.txt /requirements.txt +ADD wheelhouse /data/wheelhouse + +RUN . /data/venv/bin/activate \ +&& pip install --no-index --find-links=/data/wheelhouse -r requirements.txt \ +&& rm /requirements.txt + +VOLUME /data/git/blender-cloud +VOLUME /data/git/pillar +VOLUME /data/git/pillar-python-sdk +VOLUME /data/config +VOLUME /data/storage + +ENV USE_X_SENDFILE True + +EXPOSE 80 +EXPOSE 5000 + +ADD 000-default.conf /etc/apache2/sites-available/000-default.conf +ADD docker-entrypoint.sh /docker-entrypoint.sh +ADD manage.sh /manage.sh + +ENTRYPOINT ["bash", "/docker-entrypoint.sh"] + diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000..5594bae --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -x; +set -e; + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR; + +cd 1_base/; +bash build.sh; + +cd ../2_build/; +bash build.sh; + +cd ../3_run/; +bash build.sh; diff --git a/manage.py b/manage.py index 255a5ee..ece6bab 100755 --- a/manage.py +++ b/manage.py @@ -1,89 +1,7 @@ #!/usr/bin/env python -import logging +from pillar import cli +from cloud import app -import os -from eve.methods.post import post_internal -from eve.methods.put import put_internal -from flask.ext.script import Manager - -from pillar import PillarServer - -app = PillarServer('.') -app.process_extensions() - -# Use a sensible default when running manage.py commands. -if not os.environ.get('EVE_SETTINGS'): - settings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'pillar', 'eve_settings.py') - os.environ['EVE_SETTINGS'] = settings_path - -manager = Manager(app) -log = logging.getLogger('manage') -log.setLevel(logging.INFO) - - -def put_item(collection, item): - item_id = item['_id'] - internal_fields = ['_id', '_etag', '_updated', '_created'] - for field in internal_fields: - item.pop(field, None) - # print item - # print type(item_id) - p = put_internal(collection, item, **{'_id': item_id}) - if p[0]['_status'] == 'ERR': - print(p) - print(item) - - -@manager.command -def setup_db(admin_email): - """Setup the database - - Create admin, subscriber and demo Group collection - - Create admin user (must use valid blender-id credentials) - - Create one project - """ - - # Create default groups - groups_list = [] - for group in ['admin', 'subscriber', 'demo']: - g = {'name': group} - g = post_internal('groups', g) - groups_list.append(g[0]['_id']) - print("Creating group {0}".format(group)) - - # Create admin user - user = {'username': admin_email, - 'groups': groups_list, - 'roles': ['admin', 'subscriber', 'demo'], - 'settings': {'email_communications': 1}, - 'auth': [], - 'full_name': admin_email, - 'email': admin_email} - result, _, _, status = post_internal('users', user) - if status != 201: - raise SystemExit('Error creating user {}: {}'.format(admin_email, result)) - user.update(result) - print("Created user {0}".format(user['_id'])) - - # Create a default project by faking a POST request. - with app.test_request_context(data={'project_name': u'Default Project'}): - from flask import g - from pillar.api.projects.routes import create_project - - g.current_user = {'user_id': user['_id'], - 'groups': user['groups'], - 'roles': set(user['roles'])} - - create_project(overrides={'url': 'default-project', - 'is_private': False}) - - -@manager.command -def register_local_user(email, password): - from pillar.api.local_auth import create_local_user - create_local_user(email, password) - - -if __name__ == '__main__': - manager.run() +cli.manager.app = app +cli.manager.run() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d0efa33 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +algoliasearch==1.8.0 +bcrypt==2.0.0 +blinker==1.4 +bugsnag==2.3.1 +Cerberus==0.9.2 +Eve==0.6.3 +Events==0.2.1 +Flask==0.10.1 +Flask-Cache==0.13.1 +Flask-Login==0.3.2 +Flask-OAuthlib==0.9.3 +Flask-PyMongo==0.4.1 +Flask-Script==2.0.5 +Flask-WTF==0.12 +flup==1.0.2 +gcloud==0.12.0 +google-apitools==0.4.11 +httplib2==0.9.2 +idna==2.0 +MarkupSafe==0.23 +ndg-httpsclient==0.4.0 +Pillow==2.8.1 +pycparser==2.14 +pycrypto==2.6.1 +pyOpenSSL==0.15.1 +requests==2.9.1 +rsa==3.3 +simplejson==3.8.2 +WebOb==1.5.0 +wheel==0.24.0 +zencoder==0.6.5 diff --git a/runserver.wsgi b/runserver.wsgi index d633875..ebe2293 100644 --- a/runserver.wsgi +++ b/runserver.wsgi @@ -1,12 +1,12 @@ from os.path import abspath, dirname import sys -from pillar_server import PillarServer activate_this = '/data/venv/bin/activate_this.py' execfile(activate_this, dict(__file__=activate_this)) from flup.server.fcgi import WSGIServer +from pillar import PillarServer -sys.path.append('/data/git/blender-cloud/bcloud-server/') +sys.path.append('/data/git/blender-cloud/') application = PillarServer(dirname(abspath(__file__))) application.process_extensions()