HELK v0.1.3-alpha08032018

All + Moved all to docker folder. Getting ready to start sharing other ways to deploy helk (terraform & Packer maybe) Compose-files + Basic & Trial Elastic Subscriptions available now and can be automatically managed via the helk_install script ELK Version : 6.3.2 Elasticsearch + Set 4GB for ES_JAVA_OPTS by default allowing the modification of it via docker-compose and calculating half of the host memory if it is not set + Added Entrypoint script and using docker-entrypoint to start ES Logstash + Big Pipeline Update by Nate Guagenti (@neu5ron) ++better cli & file name searching ++”dst_ip_public:true” filter out all rfc1918/non-routable ++Geo ASName ++Identification of 16+ windows IP fields ++Arrayed IPs support ++IPv6&IPv4 differentiation ++removing “-“ values and MORE!!! ++ THANK YOU SO MUCH NATE!!! ++ PR: #93 + Added entrypoint script to push new output_templates straight to Elasticsearch per Nate's recommendation + Starting Logstash now with docker-entrypoint + "event_data" is now taken out of winlogbeat logs to allow integration with nxlog (sauce added by Nate Guagenti (@neu5ron) Kibana + Kibana yml file updated to allow a longer time for timeout Nginx: + it handles communications to Kibana and Jupyterhub via port 443 SSL + certificate and key get created at build time + Nate added several settings to improve the way how nginx operates Jupyterhub + Multiple users and mulitple notebooks open at the same time are possible now + Jupytehub now has 3 users hunter1,hunter2.hunter3 and password patterh is <user>P@ssw0rd! + Every notebook created is also JupyterLab + Updated ES-Hadoop 6.3.2 Kafka Update + 1.1.1 Update Spark Master + Brokers + reduce memory for brokers by default to 512m Resources: + Added new images for Wiki
Cyb3rWard0g · Aug 3, 2018 · 634e24e · 634e24e
1 parent c7af8e4
commit 634e24e
Show file tree

Hide file tree

Showing 164 changed files with 5,317 additions and 1,599 deletions.
diff --git a/docker-compose-elk-basic.yml → docker/docker-compose-elk-basic.yml b/docker-compose-elk-basic.yml → docker/docker-compose-elk-basic.yml
@@ -2,13 +2,15 @@ version: '3'
 
 services:
   helk-elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
     container_name: helk-elasticsearch
     volumes:
       - ./helk-elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - esdata:/usr/share/elasticsearch/data
+      - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
+    entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
     environment:
-      - "ES_JAVA_OPTS=-Xms6g -Xmx6g"
+      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
     ulimits:
       memlock:
         soft: -1
@@ -19,15 +21,17 @@ services:
         aliases:
           - helk_elasticsearch.hunt.local
   helk-logstash:
-    image: docker.elastic.co/logstash/logstash:6.3.1
+    image: docker.elastic.co/logstash/logstash:6.3.2
     container_name: helk-logstash
     volumes:
       - ./helk-logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
       - ./helk-logstash/pipeline:/usr/share/logstash/pipeline
       - ./helk-logstash/output_templates:/usr/share/logstash/output_templates
       - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
+      - ./helk-logstash/scripts:/usr/share/logstash/scripts
     environment:
-      - "LS_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "LS_JAVA_OPTS=-Xms1g -Xmx1g"
+    entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
     restart: always
     depends_on:
       - helk-elasticsearch
@@ -36,7 +40,7 @@ services:
         aliases:
           - helk_logstash.hunt.local
   helk-kibana:
-    image: docker.elastic.co/kibana/kibana:6.3.1
+    image: docker.elastic.co/kibana/kibana:6.3.2
     container_name: helk-kibana
     volumes:
       - ./helk-kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
@@ -51,26 +55,41 @@ services:
         aliases:
           - helk_kibana.hunt.local
   helk-nginx:
-    image: cyb3rward0g/helk-nginx:0.0.3
+    image: cyb3rward0g/helk-nginx:0.0.6
     container_name: helk-nginx
     volumes:
       - ./helk-nginx/htpasswd.users:/etc/nginx/htpasswd.users
       - ./helk-nginx/default:/etc/nginx/sites-available/default
+      - ./helk-nginx/scripts/:/opt/helk/scripts/
+    entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
     ports:
       - "80:80"
+      - "443:443"
     restart: always
     depends_on:
       - helk-kibana
     networks:
       helk:
         aliases:
           - helk_nginx.hunt.local
+  helk-jupyter:
+    image: cyb3rward0g/helk-jupyter:0.0.4
+    container_name: helk-jupyter
+    restart: always
+    depends_on:
+      - helk-nginx
+    networks:
+      helk:
+        aliases:
+          - helk_jupyter.hunt.local
   helk-spark-master:
-    image: cyb3rward0g/helk-spark-master:2.3.1
+    image: cyb3rward0g/helk-spark-master:2.3.1-a
     container_name: helk-spark-master
+    environment:
+      - SPARK_MASTER_PORT=7077
+      - SPARK_MASTER_WEBUI_PORT=8080
     ports:
       - "8080:8080"
-      - "7077:7077"
     restart: always
     depends_on:
       - helk-elasticsearch
@@ -79,11 +98,13 @@ services:
         aliases:
           - helk_spark_master.hunt.local
   helk-spark-worker:
-    image: cyb3rward0g/helk-spark-worker:2.3.1
+    image: cyb3rward0g/helk-spark-worker:2.3.1-a
     container_name: helk-spark-worker
     environment:
-      - SPARK_WORKER_MEMORY=1g
+      - SPARK_MASTER=spark://helk-spark-master:7077
+      - SPARK_WORKER_MEMORY=512m
       - SPARK_WORKER_WEBUI_PORT=8081
+      - SPARK_WORKER_PORT=42950
     ports:
       - "8081:8081"
     restart: always
@@ -94,11 +115,13 @@ services:
         aliases:
           - helk_spark_worker.hunt.local
   helk-spark-worker2:
-    image: cyb3rward0g/helk-spark-worker:2.3.1
+    image: cyb3rward0g/helk-spark-worker:2.3.1-a
     container_name: helk-spark-worker2
     environment:
-      - SPARK_WORKER_MEMORY=1g
+      - SPARK_MASTER=spark://helk-spark-master:7077
+      - SPARK_WORKER_MEMORY=512m
       - SPARK_WORKER_WEBUI_PORT=8082
+      - SPARK_WORKER_PORT=42951
     ports:
       - "8082:8082"
     restart: always
@@ -108,33 +131,20 @@ services:
       helk:
         aliases:
           - helk_spark_worker2.hunt.local
-  helk-jupyter:
-    image: cyb3rward0g/helk-jupyter:0.0.2
-    container_name: helk-jupyter
-    ports:
-      - "8880:8880"
-      - "4040-4050:4040-4050"
-    restart: always
-    depends_on:
-      - helk-kibana
-    networks:
-      helk:
-        aliases:
-          - helk_jupyter.hunt.local
   helk-zookeeper:
-    image: cyb3rward0g/helk-zookeeper:3.4.10
+    image: cyb3rward0g/helk-zookeeper:1.1.1
     container_name: helk-zookeeper
     ports:
       - "2181:2181"
     restart: always
     depends_on:
-      - helk-elasticsearch
+      - helk-kibana
     networks:
       helk:
         aliases:
           - helk_zookeeper.hunt.local
   helk-kafka-broker:
-    image: cyb3rward0g/helk-kafka-broker:1.1.0
+    image: cyb3rward0g/helk-kafka-broker:1.1.1
     container_name: helk-kafka-broker
     restart: always
     depends_on:
@@ -153,7 +163,7 @@ services:
         aliases:
           - helk_kafka_broker.hunt.local
   helk-kafka-broker2:
-    image: cyb3rward0g/helk-kafka-broker:1.1.0
+    image: cyb3rward0g/helk-kafka-broker:1.1.1
     container_name: helk-kafka-broker2
     restart: always
     depends_on:
@@ -170,7 +180,7 @@ services:
     networks:
       helk:
         aliases:
-          - helk_kafka_broker.hunt.local 
+          - helk_kafka_broker2.hunt.local 
   helk-sigma:
     image: thomaspatzke/helk-sigma
     container_name: helk-sigma

diff --git a/docker-compose-elk-trial.yml → docker/docker-compose-elk-trial.yml b/docker-compose-elk-trial.yml → docker/docker-compose-elk-trial.yml
@@ -7,8 +7,10 @@ services:
     volumes:
       - ./helk-elasticsearch/trial/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - esdata:/usr/share/elasticsearch/data
+      - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
+    entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
     environment:
-      - "ES_JAVA_OPTS=-Xms6g -Xmx6g"
+      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
     ulimits:
       memlock:
         soft: -1
@@ -19,15 +21,17 @@ services:
         aliases:
           - helk_elasticsearch.hunt.local
   helk-logstash:
-    image: docker.elastic.co/logstash/logstash:6.3.1
+    image: docker.elastic.co/logstash/logstash:6.3.2
     container_name: helk-logstash
     volumes:
       - ./helk-logstash/trial/logstash.yml:/usr/share/logstash/config/logstash.yml
       - ./helk-logstash/trial/pipeline:/usr/share/logstash/pipeline
       - ./helk-logstash/output_templates:/usr/share/logstash/output_templates
       - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
+      - ./helk-logstash/trial/scripts:/usr/share/logstash/scripts
     environment:
-      - "LS_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "LS_JAVA_OPTS=-Xms1g -Xmx1g"
+    entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
     restart: always
     depends_on:
       - helk-elasticsearch
@@ -36,7 +40,7 @@ services:
         aliases:
           - helk_logstash.hunt.local
   helk-kibana:
-    image: docker.elastic.co/kibana/kibana:6.3.1
+    image: docker.elastic.co/kibana/kibana:6.3.2
     container_name: helk-kibana
     volumes:
       - ./helk-kibana/trial/kibana.yml:/usr/share/kibana/config/kibana.yml
@@ -51,25 +55,40 @@ services:
         aliases:
           - helk_kibana.hunt.local
   helk-nginx:
-    image: cyb3rward0g/helk-nginx:0.0.3
+    image: cyb3rward0g/helk-nginx:0.0.6
     container_name: helk-nginx
     volumes:
       - ./helk-nginx/trial/default:/etc/nginx/sites-available/default
+      - ./helk-nginx/scripts/:/opt/helk/scripts/
+    entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
     ports:
       - "80:80"
+      - "443:443"
     restart: always
     depends_on:
       - helk-kibana
     networks:
       helk:
         aliases:
           - helk_nginx.hunt.local
+  helk-jupyter:
+    image: cyb3rward0g/helk-jupyter:0.0.4
+    container_name: helk-jupyter
+    restart: always
+    depends_on:
+      - helk-nginx
+    networks:
+      helk:
+        aliases:
+          - helk_jupyter.hunt.local
   helk-spark-master:
-    image: cyb3rward0g/helk-spark-master:2.3.1
+    image: cyb3rward0g/helk-spark-master:2.3.1-a
     container_name: helk-spark-master
+    environment:
+      - SPARK_MASTER_PORT=7077
+      - SPARK_MASTER_WEBUI_PORT=8080
     ports:
       - "8080:8080"
-      - "7077:7077"
     restart: always
     depends_on:
       - helk-elasticsearch
@@ -78,11 +97,13 @@ services:
         aliases:
           - helk_spark_master.hunt.local
   helk-spark-worker:
-    image: cyb3rward0g/helk-spark-worker:2.3.1
+    image: cyb3rward0g/helk-spark-worker:2.3.1-a
     container_name: helk-spark-worker
     environment:
-      - SPARK_WORKER_MEMORY=1g
+      - SPARK_MASTER=spark://helk-spark-master:7077
+      - SPARK_WORKER_MEMORY=512m
       - SPARK_WORKER_WEBUI_PORT=8081
+      - SPARK_WORKER_PORT=42950
     ports:
       - "8081:8081"
     restart: always
@@ -93,11 +114,13 @@ services:
         aliases:
           - helk_spark_worker.hunt.local
   helk-spark-worker2:
-    image: cyb3rward0g/helk-spark-worker:2.3.1
+    image: cyb3rward0g/helk-spark-worker:2.3.1-a
     container_name: helk-spark-worker2
     environment:
-      - SPARK_WORKER_MEMORY=1g
+      - SPARK_MASTER=spark://helk-spark-master:7077
+      - SPARK_WORKER_MEMORY=512m
       - SPARK_WORKER_WEBUI_PORT=8082
+      - SPARK_WORKER_PORT=42951
     ports:
       - "8082:8082"
     restart: always
@@ -107,21 +130,8 @@ services:
       helk:
         aliases:
           - helk_spark_worker2.hunt.local
-  helk-jupyter:
-    image: cyb3rward0g/helk-jupyter:0.0.2
-    container_name: helk-jupyter
-    ports:
-      - "8880:8880"
-      - "4040-4050:4040-4050"
-    restart: always
-    depends_on:
-      - helk-kibana
-    networks:
-      helk:
-        aliases:
-          - helk_jupyter.hunt.local
   helk-zookeeper:
-    image: cyb3rward0g/helk-zookeeper:3.4.10
+    image: cyb3rward0g/helk-zookeeper:1.1.1
     container_name: helk-zookeeper
     ports:
       - "2181:2181"
@@ -133,7 +143,7 @@ services:
         aliases:
           - helk_zookeeper.hunt.local
   helk-kafka-broker:
-    image: cyb3rward0g/helk-kafka-broker:1.1.0
+    image: cyb3rward0g/helk-kafka-broker:1.1.1
     container_name: helk-kafka-broker
     restart: always
     depends_on:
@@ -152,7 +162,7 @@ services:
         aliases:
           - helk_kafka_broker.hunt.local
   helk-kafka-broker2:
-    image: cyb3rward0g/helk-kafka-broker:1.1.0
+    image: cyb3rward0g/helk-kafka-broker:1.1.1
     container_name: helk-kafka-broker2
     restart: always
     depends_on:

diff --git a/helk-base/Dockerfile → docker/helk-base/Dockerfile b/helk-base/Dockerfile → docker/helk-base/Dockerfile
diff --git a/helk-elasticsearch/Dockerfile → docker/helk-elasticsearch/Dockerfile b/helk-elasticsearch/Dockerfile → docker/helk-elasticsearch/Dockerfile
@@ -1,12 +1,12 @@
 # HELK script: HELK Elasticsearch Dockerfile
 # HELK build Stage: Alpha
-# HELK ELK version: 6.3.1
+# HELK ELK version: 6.3.2
 # Author: Roberto Rodriguez (@Cyb3rWard0g)
 # License: GPL-3.0
 
 # References: 
 # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
 
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2
 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
 LABEL description="Dockerfile base for the HELK Elasticsearch."
diff --git a/helk-elasticsearch/elasticsearch.yml → docker/helk-elasticsearch/elasticsearch.yml b/helk-elasticsearch/elasticsearch.yml → docker/helk-elasticsearch/elasticsearch.yml
diff --git a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# HELK script: elasticsearch-entrypoint.sh
+# HELK script description: sets elasticsearch configs and starts elasticsearch
+# HELK build Stage: Alpha
+# Author: Roberto Rodriguez (@Cyb3rWard0g)
+# License: GPL-3.0
+
+# *********** Looking for ES ***************
+if [[ ! -z "$ES_JAVA_OPTS" ]]; then
+  echo "[HELK-DOCKER-INSTALLATION-INFO] Setting ES_JAVA_OPTS to $ES_JAVA_OPTS"
+else
+  # ****** Setup heap size and memory locking *****
+    ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo)
+    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting ES_HEAP_SIZE to ${ES_MEMORY}.."
+    export ES_JAVA_OPTS="-Xms${ES_MEMORY}g -Xmx${ES_MEMORY}g"
+fi
+
+# ********** Starting Elasticsearch *****************
+echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."
+/usr/local/bin/docker-entrypoint.sh
diff --git a/helk-elasticsearch/trial/Dockerfile → docker/helk-elasticsearch/trial/Dockerfile b/helk-elasticsearch/trial/Dockerfile → docker/helk-elasticsearch/trial/Dockerfile
@@ -1,15 +1,13 @@
 # HELK script: HELK Elasticsearch Dockerfile
 # HELK build Stage: Alpha
-# HELK ELK version: 6.3.1
+# HELK ELK version: 6.3.2
 # Author: Roberto Rodriguez (@Cyb3rWard0g)
 # License: GPL-3.0
 
 # References: 
 # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
 
-# *********** ELK Version ***************
-
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2
 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
 LABEL description="Dockerfile base for the HELK Elasticsearch."
 

diff --git a/helk-elasticsearch/trial/elasticsearch.yml → ...elk-elasticsearch/trial/elasticsearch.yml b/helk-elasticsearch/trial/elasticsearch.yml → ...elk-elasticsearch/trial/elasticsearch.yml