[HOT FIX] 01312019

helk ELK Updated to version 6.5.4 helk-logstash fix #156 + Pipeline Updated ++ More security events ++ Reduced regex complexity to split process paths to process names ++ Enabled Kafka output again for Win Security and Win Sysmon logs ++ Added more win security conversion events helk-elastalert fix #157 fix #159 ELK: + Consolidated ELK scripts to one per container instead of trial and basic helk-sigma + Updated own fork helk-jupyter + Updated Elastic ES-Hadoop to 6.5.4 helk-jupyter + jupyterlab-manager widgets + Updated pandas 0.24.0 + Updated altair 2.3.0
Cyb3rWard0g · Jan 31, 2019 · c7086ab · c7086ab
1 parent 9b7d224
commit c7086ab
Show file tree

Hide file tree

Showing 43 changed files with 838 additions and 2,326 deletions.
diff --git a/docker/helk-elastalert/rules/cobalt_strike_msagent.yml b/docker/helk-elastalert/rules/cobalt_strike_msagent.yml
diff --git a/docker/helk-elastalert/rules/cobalt_strike_rundll32.yml b/docker/helk-elastalert/rules/cobalt_strike_rundll32.yml
diff --git a/docker/helk-elastalert/rules/psexec_psh.yml b/docker/helk-elastalert/rules/psexec_psh.yml
diff --git a/docker/helk-elastalert/rules/whoami.yml b/docker/helk-elastalert/rules/whoami.yml
diff --git a/docker/helk-elastalert/scripts/elastalert-entrypoint.sh b/docker/helk-elastalert/scripts/elastalert-entrypoint.sh
@@ -12,65 +12,58 @@
 if [[ -z "$ES_HOST" ]]; then
   ES_HOST=helk-elasticsearch
 fi
-echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server name to $ES_HOST"
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server name to $ES_HOST"
 
 if [[ -z "$ES_PORT" ]]; then
   ES_PORT=9200
 fi
-echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server port to $ES_PORT"
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server port to $ES_PORT"
 
-if [[ "$ELASTIC_USERNAME" ]] && [[ "$ES_PASSWORD" ]]; then
+if [[ -n "$ELASTIC_PASSWORD" ]]; then
+    if [[ -z "$ELASTIC_USERNAME" ]]; then
+        ELASTIC_USERNAME=elastic
+    fi
     echo "es_username: $ELASTIC_USERNAME" >> $ESALERT_HOME/config.yaml
     echo "es_password: $ELASTIC_PASSWORD" >> $ESALERT_HOME/config.yaml
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch username to $ELASTIC_USERNAME"
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch password to $ELASTIC_PASSWORD"
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch username to $ELASTIC_USERNAME"
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch password to $ELASTIC_PASSWORD"
     ELASTICSEARCH_ACCESS=http://$ELASTIC_USERNAME:"$ELASTIC_PASSWORD"@$ES_HOST:$ES_PORT
-    if [[ "$KIBANA_HOST" ]] && [[ "$KIBANA_PORT" ]]; then
-        KIBANA=$KIBANA_HOST:$KIBANA_PORT
-    else
-        exit 1
-    fi
 else
     ELASTICSEARCH_ACCESS=http://$ES_HOST:$ES_PORT
 fi
 
 # *********** Update Elastalert Config ******************
-echo "[HELK-DOCKER-INSTALLATION-INFO] Updating Elastalert main config.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Updating Elastalert main config.."
 sed -i "s/^es_host\:.*$/es_host\: ${ES_HOST}/g" $ESALERT_HOME/config.yaml
 sed -i "s/^es_port\:.*$/es_port\: ${ES_PORT}/g" $ESALERT_HOME/config.yaml
 
 # *********** Check if Elasticsearch is up ***************
-echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.."
 until curl -s $ES_HOST:$ES_PORT -o /dev/null; do
     sleep 1
 done
 
 # *********** Creating Elastalert Status Index ***************
 response_code=$(curl -s -o /dev/null -w "%{http_code}" $ELASTICSEARCH_ACCESS/elastalert_status)
 if [[ $response_code == 404 ]]; then
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Elastalert index.."
-    if [[ "$ELASTIC_PASSWORD" ]]; then
-        # *********** Waiting for Kibana port to be up ***************
-        echo "[++] Checking to see if kibana port is up..."
-        until curl -s $KIBANA -o /dev/null; do
-            sleep 1
-        done
-        elastalert-create-index --host $ES_HOST --port $ES_PORT --username $ELASTIC_USERNAME--password $ELASTIC_PASSWORD --no-auth --no-ssl --url-prefix '' --old-index ''
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Creating Elastalert index.."
+    if [[ -n "$ELASTIC_PASSWORD" ]]; then
+        elastalert-create-index --host $ES_HOST --port $ES_PORT --username $ELASTIC_USERNAME --password $ELASTIC_PASSWORD --no-auth --no-ssl --url-prefix '' --old-index ''
     else
         elastalert-create-index --host $ES_HOST --port $ES_PORT --no-auth --no-ssl --url-prefix '' --old-index ''
     fi
 else
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Elastalert index already exists"
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Elastalert index already exists"
 fi
 
 # *********** Transform SIGMA Rules to Elastalert Signatures *************
-echo "[HELK-DOCKER-INSTALLATION-INFO] Executing pull-sigma.sh script.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Executing pull-sigma.sh script.."
 /etc/elastalert/pull-sigma.sh
 
 # *********** Setting Slack Integration *************
 rule_counter=0
 if [[ "$SLACK_WEBHOOK_URL" ]]; then
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Slack webhook url to $SLACK_WEBHOOK_URL.."
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Slack webhook url to $SLACK_WEBHOOK_URL.."
     for er in $ESALERT_HOME/rules/*; do
         priority=$(sed -n -e 's/^priority: //p' $er)
         if [[ $priority = "1" ]]; then
@@ -96,5 +89,5 @@ if [[ "$SLACK_WEBHOOK_URL" ]]; then
     echo " "
 fi
 
-echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Elastalert.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Starting Elastalert.."
 exec "$@"
diff --git a/docker/helk-elastalert/scripts/pull-sigma.sh b/docker/helk-elastalert/scripts/pull-sigma.sh
@@ -10,7 +10,7 @@
 cd $ESALERT_SIGMA_HOME
 
 # ******* Check if Elastalert rules folder has SIGMA rules ************
-echo "[+++] Checking if Elastalert rules folder has SIGMA rules.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Checking if Elastalert rules folder has SIGMA rules.."
 if ls $ESALERT_HOME/rules/ | grep -v '^helk_' >/dev/null 2>&1; then
     echo "[+++++] SIGMA rules available in rules folder.."
     SIGMA_RULES_AVAILABLE=YES
@@ -19,11 +19,11 @@ else
 fi
 
 # ******* Check if local SIGMA repo needs update *************
-echo "[+++] Fetch updates for SIGMA remote.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Fetch updates for SIGMA remote.."
 git remote update
 
 # Reference: https://stackoverflow.com/a/3278427
-echo "[+++] Checking to see if local SIGMA repo is up to date or not.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Checking to see if local SIGMA repo is up to date or not.."
 UPSTREAM=${1:-'@{u}'}
 LOCAL=$(git rev-parse @)
 REMOTE=$(git rev-parse "$UPSTREAM")
@@ -85,7 +85,7 @@ find $ESALERT_HOME/rules/ -type f -name 'sigma_sysmon_powershell_suspicious_para
 
 # ******** Deleting Empty Files ***********
 echo " "
-echo "\Removing empty files.."
+echo "Removing empty files.."
 echo "-------------------------"
 rule_counter=0
 for ef in $ESALERT_HOME/rules/* ; do 

diff --git a/docker/helk-elasticsearch/Dockerfile b/docker/helk-elasticsearch/Dockerfile
@@ -6,6 +6,6 @@
 # References: 
 # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
 
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.5.3
+FROM docker.elastic.co/elasticsearch/elasticsearch:6.5.4
 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
 LABEL description="Dockerfile base for the HELK Elasticsearch."
diff --git a/docker/helk-elasticsearch/scripts/basic/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/basic/elasticsearch-entrypoint.sh
diff --git a/...scripts/trial/elasticsearch-entrypoint.sh → ...earch/scripts/elasticsearch-entrypoint.sh b/...scripts/trial/elasticsearch-entrypoint.sh → ...earch/scripts/elasticsearch-entrypoint.sh
@@ -6,7 +6,7 @@
 # Author: Roberto Rodriguez (@Cyb3rWard0g)
 # License: GPL-3.0
 
-# *********** Looking for ES ***************
+# *********** Setting ES_JAVA_OPTS ***************
 if [[ -z "$ES_JAVA_OPTS" ]]; then
     ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo)
     if [ $ES_MEMORY -gt 31 ]; then
@@ -16,11 +16,20 @@ if [[ -z "$ES_JAVA_OPTS" ]]; then
 fi
 echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting ES_JAVA_OPTS to $ES_JAVA_OPTS"
 
-# *********** HELK ES Password ***************
-if [[ -z "$ELASTIC_PASSWORD" ]]; then
-  export ELASTIC_PASSWORD=elasticpassword
+# ******** Checking License Type ***************
+ENVIRONMENT_VARIABLES=$(env)
+XPACK_LICENSE_TYPE="$(echo $ENVIRONMENT_VARIABLES | grep -oE 'xpack.license.self_generated.type=[^ ]*' | sed s/.*=//)"
+
+# ******** Set Trial License Variables ***************
+if [[ $XPACK_LICENSE_TYPE == "trial" ]]; then
+  # *********** HELK ES Password ***************
+  if [[ -z "$ELASTIC_PASSWORD" ]]; then
+    export ELASTIC_PASSWORD=elasticpassword
+  fi
+  echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting Elastic password to $ELASTIC_PASSWORD"
 fi
-echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting Elastic password to $ELASTIC_PASSWORD"
+
+echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting Elastic license to $XPACK_LICENSE_TYPE"
 
 # ********** Starting Elasticsearch *****************
 echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."

diff --git a/docker/helk-jupyter/Dockerfile b/docker/helk-jupyter/Dockerfile
@@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 # *********** Setting Environment Variables ***************
 ENV JUPYTER_DIR=/opt/helk/jupyter
-ENV ESHADOOP_VERSION=6.5.3
+ENV ESHADOOP_VERSION=6.5.4
 ENV POSTGRESQL_VERSION=42.2.5
 
 # *********** Installing Prerequisites ***************
@@ -23,18 +23,20 @@ RUN apt-get update -qq \
   libxml2-dev libxslt1-dev zlib1g-dev postgresql \
   # ********** Install PIP Packages ************
   && sudo pip3 install --upgrade pip \
-  && pip3 install pandas==0.23.4 \
-  altair==2.2.2 \
+  && pip3 install pandas==0.24.0 \
+  altair==2.3.0 \
   jupyter==1.0.0 \
   jupyterlab==0.35.4 \
   jupyterhub==0.9.4 \
+  ipywidgets==7.4.2 \
   # *********** Setting Jupyter Hub & Jupyter **********************
   && curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - \
   && apt-get install -y --no-install-recommends nodejs \
   && npm install --production -g configurable-http-proxy \
   && jupyter labextension install @jupyterlab/[email protected] \
   && jupyter labextension install @jupyterlab/[email protected] \
   && jupyter labextension install @mflevine/[email protected] \
+  && jupyter labextension install @jupyter-widgets/[email protected] \
   && bash -c 'mkdir -pv /opt/helk/{es-hadoop,jupyter,jupyterhub}' \
   && mkdir -v /usr/local/share/jupyter/kernels/pyspark3 \
   && mkdir -v /var/log/spark \

diff --git a/docker/helk-jupyter/spark/spark-defaults.conf b/docker/helk-jupyter/spark/spark-defaults.conf
@@ -22,7 +22,7 @@ spark.executor.logs.rolling.maxRetainedFiles 20
 # Set the strategy of rolling of executor logs.
 spark.executor.logs.rolling.strategy spark.executor.logs.rolling.time.interval
 # Comma-separated list of jars to include on the driver and executor classpaths. Globs are allowed.
-spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.5.3.jar
+spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.5.4.jar
 # Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths.
 # The coordinates should be groupId:artifactId:version.
 spark.jars.packages graphframes:graphframes:0.6.0-spark2.3-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0

diff --git a/docker/helk-kibana-analysis-basic.yml b/docker/helk-kibana-analysis-basic.yml
@@ -2,14 +2,14 @@ version: '3.5'
 
 services:
   helk-elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.5.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4
     container_name: helk-elasticsearch
     secrets:
       - source: elasticsearch.yml
         target: /usr/share/elasticsearch/config/elasticsearch.yml
     volumes:
       - esdata:/usr/share/elasticsearch/data
-      - ./helk-elasticsearch//scripts/basic:/usr/share/elasticsearch/scripts
+      - ./helk-elasticsearch//scripts:/usr/share/elasticsearch/scripts
     entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
     environment:
       - cluster.name=helk-cluster
@@ -28,7 +28,7 @@ services:
     networks:
       helk:
   helk-logstash:
-    image: docker.elastic.co/logstash/logstash:6.5.3
+    image: docker.elastic.co/logstash/logstash:6.5.4
     container_name: helk-logstash
     secrets:
       - source: logstash.yml
@@ -37,24 +37,24 @@ services:
       - ./helk-logstash/pipeline:/usr/share/logstash/pipeline
       - ./helk-logstash/output_templates:/usr/share/logstash/output_templates
       - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
-      - ./helk-logstash/scripts/basic:/usr/share/logstash/scripts
+      - ./helk-logstash/scripts:/usr/share/logstash/scripts
     entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
     ports:
       - "5044:5044"
     restart: always
     depends_on:
-      - helk-zookeeper
+      - helk-kibana
     networks:
       helk:
   helk-kibana:
-    image: docker.elastic.co/kibana/kibana:6.5.3
+    image: docker.elastic.co/kibana/kibana:6.5.4
     container_name: helk-kibana
     secrets:
       - source: kibana.yml
         target: /usr/share/kibana/config/kibana.yml
     volumes:
       - ./helk-kibana/dashboards:/usr/share/kibana/dashboards
-      - ./helk-kibana/scripts/basic:/usr/share/kibana/scripts
+      - ./helk-kibana/scripts:/usr/share/kibana/scripts
     entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh
     restart: always
     depends_on:
@@ -84,7 +84,7 @@ services:
     container_name: helk-zookeeper
     restart: always
     depends_on:
-      - helk-kibana
+      - helk-logstash
     networks:
       helk:
   helk-kafka-broker:
@@ -100,7 +100,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat
+      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, winsysmon, winsecurity
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4 
     ports:
@@ -130,7 +130,6 @@ services:
     image: confluentinc/cp-ksql-cli:5.1.0
     container_name: helk-ksql-cli
     depends_on:
-      - helk-kafka-broker
       - helk-ksql-server
     environment:
         KSQL_HEAP_OPTS: -Xmx1g
@@ -139,12 +138,11 @@ services:
     networks:
       helk:
   helk-elastalert:
-    image: cyb3rward0g/helk-elastalert:0.2.0
+    image: cyb3rward0g/helk-elastalert:0.2.1
     container_name: helk-elastalert
     restart: always
     depends_on:
-      - helk-elasticsearch
-      - helk-kibana
+      - helk-logstash
     environment:
       ES_HOST: helk-elasticsearch
       ES_PORT: 9200