v0.1.3-alpha08242018

Docker-compose Files Version
+ Updated version to 3.5

Base Docker Ubuntu Image
+ Updated to phusion/baseimage version 0.11 (https://github.com/phusion/baseimage-docker/releases/tag/0.11)

HELK base image
+ Updated to 0.0.2 due to Ubuntu upgrade

HELK ELK Version
+ Now using 6.4.0 official ELK Docker Images (https://www.elastic.co/blog/elastic-stack-6-4-0-released?blade=tw&hulk=social)

helk_install
+ Fixed #99

helk-elasticsearch
+ Updated main yml config to set most of the settings via environment variables via docker-compose
+ Trial docker-compose file now has ELASTICSEARCH_PASSWORD environment variable set/available. Trial Dockerfile was deleted since the elasticsearch_password update is now taken care of by the internal elasticsearch docker script that is comes with the official elasticsearch docker image.
+ reduced the memory requirements from 4GB to 2GB

helk-logstash
+ entrypoint scripts remove kafka output plugin 7.1.2 and installs version 7.1.1 due to logstash-plugins/logstash-output-kafka#198
++ this error happens right after upgrading ELK built from 6.3.2 to 6.4.0

helk-jupyter
+ Added Altair python package
+ updated Jupyterlab to 0.34.1
+ updated jupyterhub to 0.9.2
+ updated jupyterlab hub extension to 0.11.0
+ updated Spark config to use Graphframes 0.6.0 (https://graphframes.github.io/user-guide.html)
+ updated spark-kafka library to spark-sql-kafka-0-10_2.11:2.3.1

helk-kafka-base
+ updated Kafka to 2.0.0 (this affects Kafka brokers and zookeeper)
+ Created user kafkauser to run kafka containers as non-root

helk-kafka-broker
+ split entrypoint script to have topics creation separate
++ auomated the way how the container checks for the kafka broker port availability. If the port is open, then it attempts to create kafka topics
+ No need to tail kafka logs to keep the container alive after running the kafka start script. It now just starts the broker via Dockerfile CMD command and stays alive.

helk-zookeeper
+ updated entrypoint to only set the main server config
+ zookeeper is now started via Dockerfile CMD command

docker/docker-compose-elk-basic.yml → docker/docker-compose-helk-elastic-basic.yml

@@ -1,49 +1,58 @@
-version: '3'
+version: '3.5'
 
 services:
   helk-elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0
     container_name: helk-elasticsearch
+    secrets:
+      - source: elasticsearch.yml
+        target: /usr/share/elasticsearch/config/elasticsearch.yml
     volumes:
-      - ./helk-elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - esdata:/usr/share/elasticsearch/data
       - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
     entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
     environment:
-      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
+      - cluster.name=helk-cluster
+      - node.name=helk-1
+      - bootstrap.memory_lock=true
+      - discovery.zen.minimum_master_nodes=1
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
     ulimits:
       memlock:
         soft: -1
         hard: -1
     restart: always
     networks:
       helk:
-        aliases:
-          - helk_elasticsearch.hunt.local
   helk-logstash:
-    image: docker.elastic.co/logstash/logstash:6.3.2
+    image: docker.elastic.co/logstash/logstash:6.4.0
     container_name: helk-logstash
+    secrets:
+      - source: logstash.yml
+        target: /usr/share/logstash/config/logstash.yml
     volumes:
-      - ./helk-logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
       - ./helk-logstash/pipeline:/usr/share/logstash/pipeline
       - ./helk-logstash/output_templates:/usr/share/logstash/output_templates
       - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
       - ./helk-logstash/scripts:/usr/share/logstash/scripts
     environment:
       - "LS_JAVA_OPTS=-Xms1g -Xmx1g"
     entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
+    ports:
+      - "5044:5044"
     restart: always
     depends_on:
-      - helk-elasticsearch
+      - helk-zookeeper
     networks:
       helk:
-        aliases:
-          - helk_logstash.hunt.local
   helk-kibana:
-    image: docker.elastic.co/kibana/kibana:6.3.2
+    image: docker.elastic.co/kibana/kibana:6.4.0
     container_name: helk-kibana
+    secrets:
+      - source: kibana.yml
+        target: /usr/share/kibana/config/kibana.yml
     volumes:
-      - ./helk-kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
       - ./helk-kibana/dashboards:/usr/share/kibana/dashboards
       - ./helk-kibana/scripts:/usr/share/kibana/scripts
     entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh
@@ -52,13 +61,13 @@ services:
       - helk-elasticsearch
     networks:
       helk:
-        aliases:
-          - helk_kibana.hunt.local
   helk-nginx:
-    image: cyb3rward0g/helk-nginx:0.0.6
+    image: cyb3rward0g/helk-nginx:0.0.7
     container_name: helk-nginx
+    secrets:
+      - source: htpasswd.users
+        target: /etc/nginx/htpasswd.users
     volumes:
-      - ./helk-nginx/htpasswd.users:/etc/nginx/htpasswd.users
       - ./helk-nginx/default:/etc/nginx/sites-available/default
       - ./helk-nginx/scripts/:/opt/helk/scripts/
     entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
@@ -68,22 +77,19 @@ services:
     restart: always
     depends_on:
       - helk-kibana
+      - helk-jupyter
     networks:
       helk:
-        aliases:
-          - helk_nginx.hunt.local
   helk-jupyter:
-    image: cyb3rward0g/helk-jupyter:0.0.4
+    image: cyb3rward0g/helk-jupyter:0.0.5
     container_name: helk-jupyter
     restart: always
     depends_on:
-      - helk-nginx
+      - helk-elasticsearch
     networks:
       helk:
-        aliases:
-          - helk_jupyter.hunt.local
   helk-spark-master:
-    image: cyb3rward0g/helk-spark-master:2.3.1-a
+    image: cyb3rward0g/helk-spark-master:2.3.1-b
     container_name: helk-spark-master
     environment:
       - SPARK_MASTER_PORT=7077
@@ -95,44 +101,34 @@ services:
       - helk-elasticsearch
     networks:
       helk:
-        aliases:
-          - helk_spark_master.hunt.local
   helk-spark-worker:
-    image: cyb3rward0g/helk-spark-worker:2.3.1-a
+    image: cyb3rward0g/helk-spark-worker:2.3.1-b
     container_name: helk-spark-worker
     environment:
       - SPARK_MASTER=spark://helk-spark-master:7077
       - SPARK_WORKER_MEMORY=512m
       - SPARK_WORKER_WEBUI_PORT=8081
       - SPARK_WORKER_PORT=42950
-    ports:
-      - "8081:8081"
     restart: always
     depends_on:
       - helk-spark-master
     networks:
       helk:
-        aliases:
-          - helk_spark_worker.hunt.local
   helk-spark-worker2:
-    image: cyb3rward0g/helk-spark-worker:2.3.1-a
+    image: cyb3rward0g/helk-spark-worker:2.3.1-b
     container_name: helk-spark-worker2
     environment:
       - SPARK_MASTER=spark://helk-spark-master:7077
       - SPARK_WORKER_MEMORY=512m
       - SPARK_WORKER_WEBUI_PORT=8082
       - SPARK_WORKER_PORT=42951
-    ports:
-      - "8082:8082"
     restart: always
     depends_on:
       - helk-spark-master
     networks:
       helk:
-        aliases:
-          - helk_spark_worker2.hunt.local
   helk-zookeeper:
-    image: cyb3rward0g/helk-zookeeper:1.1.1
+    image: cyb3rward0g/helk-zookeeper:2.0.0-a
     container_name: helk-zookeeper
     ports:
       - "2181:2181"
@@ -141,10 +137,8 @@ services:
       - helk-kibana
     networks:
       helk:
-        aliases:
-          - helk_zookeeper.hunt.local
   helk-kafka-broker:
-    image: cyb3rward0g/helk-kafka-broker:1.1.1
+    image: cyb3rward0g/helk-kafka-broker:2.0.0-b
     container_name: helk-kafka-broker
     restart: always
     depends_on:
@@ -156,14 +150,13 @@ services:
       - REPLICATION_FACTOR=2
       - ADVERTISED_LISTENER=HOSTIP
       - ZOOKEEPER_NAME=helk-zookeeper
+      - KAFKA_CREATE_TOPICS=True
     ports:
       - "9092:9092"
     networks:
       helk:
-        aliases:
-          - helk_kafka_broker.hunt.local
   helk-kafka-broker2:
-    image: cyb3rward0g/helk-kafka-broker:1.1.1
+    image: cyb3rward0g/helk-kafka-broker:2.0.0-b
     container_name: helk-kafka-broker2
     restart: always
     depends_on:
@@ -175,25 +168,26 @@ services:
       - REPLICATION_FACTOR=2
       - ADVERTISED_LISTENER=HOSTIP
       - ZOOKEEPER_NAME=helk-zookeeper
+      - KAFKA_CREATE_TOPICS=True
     ports:
       - "9093:9093"
     networks:
       helk:
-        aliases:
-          - helk_kafka_broker2.hunt.local 
-  helk-sigma:
-    image: thomaspatzke/helk-sigma
-    container_name: helk-sigma
-    depends_on:
-      - helk-kibana
-    networks:
-      helk:
-        aliases:
-          - helk_sigma.hunt.local
+
 networks:
   helk:
     driver: bridge
 
 volumes:
   esdata:
-    driver: local
+    driver: local
+
+secrets:
+  elasticsearch.yml:
+    file: ./helk-elasticsearch/elasticsearch.yml
+  logstash.yml:
+    file: ./helk-logstash/logstash.yml
+  kibana.yml:
+    file: ./helk-kibana/kibana.yml
+  htpasswd.users:
+    file: ./helk-nginx/htpasswd.users