Placing code + assets directly into Docker image

This radically changes the way we deploy to the production server, as a Git checkout is no longer required there. All the necessary files are now inside the docker image. As a result, /data/git should no longer be mounted as a Docker volume. - Renamed docker/build.sh → docker/full_rebuild.sh This makes it clearer that it performs a full rebuild of the Docker images. - Full rebuilds should be done on a regular basis to pull in Ubuntu security updates. - Removed rsync_ui.sh, we no longer need it. Other projects can also remove their rsync_ui.sh. - Moved deploy.sh → deploy/2docker.sh and added deploy/2server.sh
2018-02-01 15:13:05 +01:00
parent ffc4f271e8
commit 94ef616593
10 changed files with 387 additions and 278 deletions
--- a/docker/4_run/config_local.py
+++ b/docker/4_run/config_local.py
@@ -0,0 +1,126 @@
+import os
+from collections import defaultdict
+DEBUG = False
+
+SCHEME = 'https'
+PREFERRED_URL_SCHEME = 'https'
+SERVER_NAME = 'cloud.blender.org'
+
+#os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = 'true'
+os.environ['PILLAR_MONGO_DBNAME'] = 'cloud'
+os.environ['PILLAR_MONGO_PORT'] = '27017'
+os.environ['PILLAR_MONGO_HOST'] = 'mongo'
+
+USE_X_SENDFILE = True
+
+STORAGE_BACKEND = 'gcs'
+
+CDN_SERVICE_DOMAIN = 'blendercloud-pro.r.worldssl.net'
+CDN_CONTENT_SUBFOLDER = ''
+CDN_STORAGE_ADDRESS = 'push-11.cdnsun.com'
+
+CACHE_TYPE = 'redis' #null
+CACHE_KEY_PREFIX = 'pw_'
+CACHE_REDIS_HOST = 'redis'
+CACHE_REDIS_PORT = '6379'
+CACHE_REDIS_URL = 'redis://redis:6379'
+
+PILLAR_SERVER_ENDPOINT = 'https://cloud.blender.org/api/'
+
+BLENDER_ID_ENDPOINT = 'https://www.blender.org/id/'
+
+GCLOUD_APP_CREDENTIALS = '/data/config/google_app.json'
+GCLOUD_PROJECT = 'blender-cloud'
+
+MAIN_PROJECT_ID = '563a9c8cf0e722006ce97b03'
+# MAIN_PROJECT_ID = '57aa07c088bef606e89078bd'
+
+ALGOLIA_INDEX_USERS = 'pro_Users'
+ALGOLIA_INDEX_NODES = 'pro_Nodes'
+
+ZENCODER_NOTIFICATIONS_URL = 'https://cloud.blender.org/api/encoding/zencoder/notifications'
+
+FILE_LINK_VALIDITY = defaultdict(
+    lambda: 3600 * 24 * 30,  # default of 1 month.
+    gcs=3600 * 23,  # 23 hours for Google Cloud Storage.
+    cdnsun=3600 * 23
+)
+
+LOGGING = {
+    'version': 1,
+    'formatters': {
+        'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'default',
+            'stream': 'ext://sys.stderr',
+        }
+    },
+    'loggers': {
+        'pillar': {'level': 'INFO'},
+        #'pillar.auth': {'level': 'DEBUG'},
+        #'pillar.api.blender_id': {'level': 'DEBUG'},
+        #'pillar.api.blender_cloud.subscription': {'level': 'DEBUG'},
+        'bcloud': {'level': 'INFO'},
+        'cloud': {'level': 'INFO'},
+        'attract': {'level': 'INFO'},
+        'flamenco': {'level': 'INFO'},
+        #'pillar.api.file_storage': {'level': 'DEBUG'},
+        #'pillar.api.file_storage.ensure_valid_link': {'level': 'INFO'},
+        'pillar.api.file_storage.refresh_links_for_backend': {'level': 'DEBUG'},
+        'werkzeug': {'level': 'DEBUG'},
+        'eve': {'level': 'WARNING'},
+        #'elasticsearch': {'level': 'DEBUG'},
+    },
+    'root': {
+        'level': 'WARNING',
+        'handlers': [
+            'console',
+        ],
+    }
+}
+
+REDIRECTS = {
+    # For old links, refer to the services page (hopefully it refreshes then)
+    'downloads/blender_cloud-latest-bundle.zip': 'https://cloud.blender.org/services#blender-addon',
+
+    # Latest Blender Cloud add-on; remember to update BLENDER_CLOUD_ADDON_VERSION.
+    'downloads/blender_cloud-latest-addon.zip':
+        'https://storage.googleapis.com/institute-storage/addons/blender_cloud-1.8.0.addon.zip',
+
+    # Redirect old Grafista endpoint to /stats
+    '/stats/': '/stats',
+}
+
+# Latest version of the add-on; remember to update REDIRECTS.
+BLENDER_CLOUD_ADDON_VERSION = '1.8.0'
+
+UTM_LINKS = {
+    'cartoon_brew': {
+        'image': 'https://imgur.com/13nQTi3.png',
+        'link': 'https://store.blender.org/product/membership/'
+    }
+}
+
+# Disabled until we have regenerated the majority of the links.
+CELERY_BEAT_SCHEDULE = {
+    'regenerate-expired-links': {
+        'task': 'pillar.celery.file_link_tasks.regenerate_all_expired_links',
+        'schedule': 600,  # every N seconds
+        'args': ('gcs', 500)
+    },
+}
+
+SVNMAN_REPO_URL = 'https://svn.blender.cloud/repo/'
+SVNMAN_API_URL = 'https://svn.blender.cloud/api/'
+
+# Mail options, see pillar.celery.email_tasks.
+SMTP_HOST = 'proog.blender.org'
+SMTP_PORT = 25
+SMTP_USERNAME = 'server@blender.cloud'
+SMTP_TIMEOUT = 30  # timeout in seconds, https://docs.python.org/3/library/smtplib.html#smtplib.SMTP
+MAIL_RETRY = 180  # in seconds, delay until trying to send an email again.
+MAIL_DEFAULT_FROM_NAME = 'Blender Cloud'
+MAIL_DEFAULT_FROM_ADDR = 'cloudsupport@blender.org'
--- a/docker/4_run/run.docker
+++ b/docker/4_run/run.docker
@@ -27,7 +27,9 @@ RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR $APACHE_LOG_DIR
 ADD wheelhouse /data/wheelhouse
 RUN pip3 install --no-index --find-links=/data/wheelhouse -r /data/wheelhouse/requirements.txt

-VOLUME /data/git
+ADD deploy /data/git
+ADD config_local.py /data/git/blender-cloud/
+
 VOLUME /data/config
 VOLUME /data/storage
 VOLUME /var/log
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -9,6 +9,11 @@ services:
      - /data/storage/db-bak:/data/db-bak # for backing up stuff etc.
    ports:
      - "127.0.0.1:27017:27017"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  redis:
    image:                   redis:3.2.8
@@ -16,6 +21,11 @@ services:
    restart:                 always
    ports:
      - "127.0.0.1:6379:6379"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  rabbit:
    image:                   rabbitmq:3.6.10
@@ -23,6 +33,11 @@ services:
    restart:                 always
    ports:
      - "127.0.0.1:5672:5672"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  elastic:
    image:                   armadillica/elasticsearch:6.1.1
@@ -35,6 +50,11 @@ services:
      - "127.0.0.1:9200:9200"
    environment:
      ES_JAVA_OPTS:          "-Xms256m -Xmx256m"
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  elasticproxy:
    image:                   armadillica/elasticproxy:1.2
@@ -43,6 +63,11 @@ services:
    command:                 /elasticproxy -elastic http://elastic:9200/
    depends_on:
      - elastic
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  kibana:
    image:                   armadillica/kibana:6.1.1
@@ -60,6 +85,11 @@ services:
      NODE_OPTIONS:          "--max-old-space-size=200"
    depends_on:
      - elasticproxy
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  blender_cloud:
    image:                   armadillica/blender_cloud:latest
@@ -70,9 +100,9 @@ services:
      VIRTUAL_HOST_WEIGHT:   10
      FORCE_SSL:             "true"
      GZIP_COMPRESSION_TYPE: "text/html text/plain text/css application/javascript"
+      PILLAR_CONFIG:         /data/config/config_secrets.py
    volumes:
      # format:              HOST:CONTAINER
-      - /data/git:/data/git:ro
      - /data/config:/data/config:ro
      - /data/storage/pillar:/data/storage/pillar
      - /data/log:/var/log
@@ -86,9 +116,10 @@ services:
    entrypoint:              /celery-worker.sh
    container_name:          celery_worker
    restart:                 always
+    environment:
+      PILLAR_CONFIG:         /data/config/config_secrets.py
    volumes:
      # format:              HOST:CONTAINER
-      - /data/git:/data/git:ro
      - /data/config:/data/config:ro
      - /data/storage/pillar:/data/storage/pillar
      - /data/log:/var/log
@@ -96,34 +127,34 @@ services:
      - mongo
      - redis
      - rabbit
-
-  celery_beat:
-    image:                   armadillica/blender_cloud:latest
-    entrypoint:              /celery-beat.sh
-    container_name:          celery_beat
-    restart:                 always
-    volumes:
-      # format:              HOST:CONTAINER
-      - /data/git:/data/git:ro
-      - /data/storage/pillar:/data/storage/pillar
-      - /data/log:/var/log
-    depends_on:
-      - mongo
-      - redis
-      - rabbit
    logging:
      driver:                "json-file"
      options:
        max-size:            "200k"
        max-file:            "20"

-  grafista:
-    image:                   armadillica/grafista:latest
-    container_name:          grafista
+  celery_beat:
+    image:                   armadillica/blender_cloud:latest
+    entrypoint:              /celery-beat.sh
+    container_name:          celery_beat
    restart:                 always
+    environment:
+      PILLAR_CONFIG:         /data/config/config_secrets.py
    volumes:
-      - /data/git/grafista:/data/git/grafista:ro
-      - /data/storage/grafista:/data/storage/grafista
+      # format:              HOST:CONTAINER
+      - /data/config:/data/config:ro
+      - /data/storage/pillar:/data/storage/pillar
+      - /data/log:/var/log
+    depends_on:
+      - mongo
+      - redis
+      - rabbit
+      - celery_worker
+    logging:
+      driver:                "json-file"
+      options:
+        max-size:            "200k"
+        max-file:            "20"

  letsencrypt:
    image:                   armadillica/picohttp:latest
--- a/docker/full_rebuild.sh
+++ b/docker/full_rebuild.sh