diff --git a/docker/README.md b/docker/README.md index c990f21..7b5762a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -68,7 +68,15 @@ Blender Cloud expects the following files to exist: ## 7. ElasticSearch & kibana -Kibana should be placed in read-only mode: +ElasticSearch and Kibana run in our self-rolled images. This is needed because by default + +- ElasticSearch uses up to 2 GB of RAM, which is too much for our droplet, and +- the Docker images contain the proprietary X-Pack plugin, which we don't want. + +This also gives us the opportunity to let Kibana do its optimization when we build the image, rather +than every time the container is recreated. + +Production Kibana should be placed in read-only mode: `curl -XPUT 'localhost:9200/.kibana/_settings' -d '{ "index.blocks.read_only" : true }'` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 6781fc2..4685775 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,7 @@ rabbit: ports: - "127.0.0.1:5672:5672" elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.1 + image: armadillica/elasticsearch:latest container_name: elastic restart: always volumes: @@ -28,10 +28,8 @@ elastic: - /data/storage/elastic:/usr/share/elasticsearch/data ports: - "127.0.0.1:9200:9200" - environment: - xpack.security.enabled: 'false' kibana: - image: docker.elastic.co/kibana/kibana:5.6.1 + image: armadillica/kibana:latest container_name: kibana restart: always environment: diff --git a/docker/elastic/Dockerfile-elastic b/docker/elastic/Dockerfile-elastic new file mode 100644 index 0000000..7294104 --- /dev/null +++ b/docker/elastic/Dockerfile-elastic @@ -0,0 +1,10 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.1 +LABEL maintainer Sybren A. Stüvel + +RUN elasticsearch-plugin remove --purge x-pack + +ADD elasticsearch.yml jvm.options /usr/share/elasticsearch/config/ + +USER root +RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config/ +USER elasticsearch diff --git a/docker/elastic/Dockerfile-kibana b/docker/elastic/Dockerfile-kibana new file mode 100644 index 0000000..be98dc9 --- /dev/null +++ b/docker/elastic/Dockerfile-kibana @@ -0,0 +1,5 @@ +FROM docker.elastic.co/kibana/kibana:5.6.1 +LABEL maintainer Sybren A. Stüvel + +RUN bin/kibana-plugin remove x-pack +RUN kibana 2>&1 | grep -m 1 "Optimization of .* complete" diff --git a/docker/elastic/build.sh b/docker/elastic/build.sh new file mode 100755 index 0000000..1be711c --- /dev/null +++ b/docker/elastic/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +docker build -t armadillica/elasticsearch:latest -f Dockerfile-elastic . +docker build -t armadillica/kibana:latest -f Dockerfile-kibana . + +echo "Done, built armadillica/elasticsearch:latest and armadillica/kibana:latest" diff --git a/docker/elastic/elasticsearch.yml b/docker/elastic/elasticsearch.yml new file mode 100644 index 0000000..c0385d4 --- /dev/null +++ b/docker/elastic/elasticsearch.yml @@ -0,0 +1,7 @@ +cluster.name: "blender-cloud" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# set to 1 to allow single node clusters +# Details: https://github.com/elastic/elasticsearch/pull/17288 +discovery.zen.minimum_master_nodes: 1 diff --git a/docker/elastic/jvm.options b/docker/elastic/jvm.options new file mode 100644 index 0000000..490fef1 --- /dev/null +++ b/docker/elastic/jvm.options @@ -0,0 +1,112 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +# Sybren: changed from 2g to 512m +-Xms512m +-Xmx512m + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM (remove on 32-bit client JVMs) +-server + +# explicitly set the stack size (reduce to 320k on 32-bit client JVMs) +-Xss1m + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# use old-style file permissions on JDK9 +-Djdk.io.permissionsUseCanonicalPath=true + +# flags to configure Netty +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true +-Dio.netty.recycler.maxCapacityPerThread=0 + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Dlog4j.skipJansi=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + +# By default, the GC log file will not rotate. +# By uncommenting the lines below, the GC log file +# will be rotated every 128MB at most 32 times. +#-XX:+UseGCLogFileRotation +#-XX:NumberOfGCLogFiles=32 +#-XX:GCLogFileSize=128M + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true