[HOT-FIX] 02022019

helk-Elasticsearch - Adjusted ES JAVA OPTs (Heap size) calculations helk-jupyter + Upgraded image to 0.1.0 + Updated graphframes to 0.7.0 + fix #161 + fix #163 helk-logstash + fix #162
Cyb3rWard0g · Feb 2, 2019 · 4184706 · 4184706
1 parent 5318e9e
commit 4184706
Show file tree

Hide file tree

Showing 11 changed files with 111 additions and 32 deletions.
diff --git a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh
@@ -8,9 +8,16 @@
 
 # *********** Setting ES_JAVA_OPTS ***************
 if [[ -z "$ES_JAVA_OPTS" ]]; then
-    ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo)
-    if [ $ES_MEMORY -gt 31 ]; then
-      ES_MEMORY=31
+    AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo)
+    if [ $AVAILABLE_MEMORY -ge 8 -a $AVAILABLE_MEMORY -le 12 ]; then
+      ES_MEMORY=2
+    elif [$AVAILABLE_MEMORY -ge 13 -a $AVAILABLE_MEMORY -le 16]; then
+      ES_MEMORY=4
+    else
+      ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo)
+      if [ $ES_MEMORY -gt 31 ]; then
+        ES_MEMORY=31
+      fi
     fi
     export ES_JAVA_OPTS="-Xms${ES_MEMORY}g -Xmx${ES_MEMORY}g"
 fi

diff --git a/docker/helk-jupyter/Dockerfile b/docker/helk-jupyter/Dockerfile
@@ -29,6 +29,11 @@ RUN apt-get update -qq \
   jupyterlab==0.35.4 \
   jupyterhub==0.9.4 \
   ipywidgets==7.4.2 \
+  matplotlib==3.0.2 \
+  scipy==1.2.0 \
+  scikit-learn==0.20.2 \
+  Keras==2.2.4 \
+  s3fs==0.2.0 \
   # *********** Setting Jupyter Hub & Jupyter **********************
   && curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - \
   && apt-get install -y --no-install-recommends nodejs \

diff --git a/docker/helk-jupyter/kernels/pyspark_kernel.json b/docker/helk-jupyter/kernels/pyspark_kernel.json
@@ -11,7 +11,6 @@
   "env": {
    "SPARK_HOME": "/opt/helk/spark/",
    "PYTHONPATH": "/opt/helk/spark/python/:/opt/helk/spark/python/lib/py4j-0.10.7-src.zip",
-   "PYTHONSTARTUP": "/opt/helk/spark/python/pyspark/shell.py",
    "PYSPARK_PYTHON": "/usr/bin/python3"
   }
 }
diff --git a/docker/helk-jupyter/scripts/jupyter-entrypoint.sh b/docker/helk-jupyter/scripts/jupyter-entrypoint.sh
@@ -37,7 +37,7 @@ if [[ $HELK_USER_EXISTS == "1" ]]; then
     echo "[HELK-JUPYTER-DOCKER-INSTALLATION-INFO] Creating JupyterHub Group..."
     groupadd -g ${JUPYTERHUB_GID} jupyterhub
 
-    # ************* Create notebooks folder if it is not provided in comose file ******************
+    # ************* Create notebooks folder if it is not provided in compose file ******************
     mkdir -p ${JUPYTER_NOTEBOOKS}
 
     # ************* Creating JupyterHub Admin ***************

diff --git a/docker/helk-jupyter/hive-site.xml → docker/helk-jupyter/spark/hive-site.xml b/docker/helk-jupyter/hive-site.xml → docker/helk-jupyter/spark/hive-site.xml
diff --git a/docker/helk-jupyter/spark/spark-defaults.conf b/docker/helk-jupyter/spark/spark-defaults.conf
@@ -12,7 +12,7 @@
 # Logs the effective SparkConf as INFO when a SparkContext is started. Default: false
 spark.logConf true
 # The cluster manager to connect to.
-spark.master spark://helk-spark-master:7077
+# spark.master spark://helk-spark-master:7077
 # Restarts the driver automatically if it fails with a non-zero exit status
 spark.driver.supervise true
 
@@ -25,7 +25,7 @@ spark.executor.logs.rolling.strategy spark.executor.logs.rolling.time.interval
 spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.5.4.jar
 # Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths.
 # The coordinates should be groupId:artifactId:version.
-spark.jars.packages graphframes:graphframes:0.6.0-spark2.3-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0
+spark.jars.packages graphframes:graphframes:0.7.0-spark2.4-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0
 #spark.jars.packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1,databricks:spark-sklearn:0.2.3
 
 # ************ Spark UI ****************

diff --git a/docker/helk-kibana-notebook-analysis-basic.yml b/docker/helk-kibana-notebook-analysis-basic.yml
@@ -81,13 +81,13 @@ services:
     networks:
       helk:
   helk-jupyter:
-    image: cyb3rward0g/helk-jupyter:0.0.9
+    image: cyb3rward0g/helk-jupyter:0.1.0
     container_name: helk-jupyter
     volumes:
       - ./helk-jupyter/notebooks:/opt/helk/jupyter/notebooks
     environment:
       JUPYTER_HELK_PWD: hunting
-      JUPYTER_USERS: hunter1, hunter2
+      JUPYTER_USERS: hunter1
     restart: always
     depends_on:
       - helk-logstash
@@ -161,7 +161,7 @@ services:
       KSQL_KSQL_COMMIT_INTERVAL_MS: 2000
       KSQL_KSQL_CACHE_MAX_BYTES_BUFFERING: 10000000
       KSQL_KSQL_STREAMS_AUTO_OFFSET_RESET: earliest
-      KSQL_HEAP_OPTS: -Xmx1g
+      KSQL_HEAP_OPTS: -Xmx500m
     ports:
       - 8088:8088
     networks:
@@ -172,7 +172,7 @@ services:
     depends_on:
       - helk-ksql-server
     environment:
-        KSQL_HEAP_OPTS: -Xmx550m
+        KSQL_HEAP_OPTS: -Xmx500m
     entrypoint: /bin/sh
     tty: true
     networks:

diff --git a/docker/helk-kibana-notebook-analysis-trial.yml b/docker/helk-kibana-notebook-analysis-trial.yml
@@ -83,13 +83,13 @@ services:
     networks:
       helk:
   helk-jupyter:
-    image: cyb3rward0g/helk-jupyter:0.0.9
+    image: cyb3rward0g/helk-jupyter:0.1.0
     container_name: helk-jupyter
     volumes:
       - ./helk-jupyter/notebooks:/opt/helk/jupyter/notebooks
     environment:
       JUPYTER_HELK_PWD: hunting
-      JUPYTER_USERS: hunter1, hunter2
+      JUPYTER_USERS: hunter1
     restart: always
     depends_on:
       - helk-logstash
@@ -143,7 +143,7 @@ services:
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
       KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, winsysmon, winsecurity
-      KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
+      KAFKA_HEAP_OPTS: -Xmx1g -Xms1g
       LOG_RETENTION_HOURS: 4 
     ports:
       - "9092:9092"
@@ -163,7 +163,7 @@ services:
       KSQL_KSQL_COMMIT_INTERVAL_MS: 2000
       KSQL_KSQL_CACHE_MAX_BYTES_BUFFERING: 10000000
       KSQL_KSQL_STREAMS_AUTO_OFFSET_RESET: earliest
-      KSQL_HEAP_OPTS: -Xmx1g
+      KSQL_HEAP_OPTS: -Xmx500m
     ports:
       - 8088:8088
     networks:
@@ -174,7 +174,7 @@ services:
     depends_on:
       - helk-ksql-server
     environment:
-        KSQL_HEAP_OPTS: -Xmx550m
+        KSQL_HEAP_OPTS: -Xmx500m
     entrypoint: /bin/sh
     tty: true
     networks:

diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh
@@ -79,13 +79,6 @@ for file in ${DIR}/*.json; do
     done
 done
 
-# ********* Setting LS_JAVA_OPTS ***************
-if [[ -z "$LS_JAVA_OPTS" ]]; then
-    LS_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/4}' /proc/meminfo)
-    export LS_JAVA_OPTS="-Xms${LS_MEMORY}m -Xmx${LS_MEMORY}m"
-fi
-echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Setting LS_JAVA_OPTS to $LS_JAVA_OPTS"
-
 # ********** Install Plugin *****************
 echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.."
 if logstash-plugin list 'prune'; then
@@ -94,6 +87,21 @@ else
     logstash-plugin install logstash-filter-prune
 fi
 
+# ********* Setting LS_JAVA_OPTS ***************
+if [[ -z "$LS_JAVA_OPTS" ]]; then
+  while true; do
+    LS_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/4}' /proc/meminfo)
+    if [ $LS_MEMORY -gt 980 ]; then
+      export LS_JAVA_OPTS="-Xms${LS_MEMORY}m -Xmx${LS_MEMORY}m"
+      break
+    else
+      echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] $LS_MEMORY MB is not enough memory for Logstash yet.."
+      sleep 1
+    fi
+  done
+fi
+echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Setting LS_JAVA_OPTS to $LS_JAVA_OPTS"
+
 # ********** Starting Logstash *****************
 echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."
 /usr/local/bin/docker-entrypoint
diff --git a/docker/helk_docker_install.sh b/docker/helk_docker_install.sh
@@ -19,20 +19,80 @@ echoerror() {
 }
 
 # ********* Globals **********************
-systemKernel="$(uname -s)"
+SYSTEM_KERNEL="$(uname -s)"
+
+echo "[HELK-DOCKER-INSTALLATION-INFO] Checking distribution list and product version"
+if [ "$SYSTEM_KERNEL" == "Linux" ]; then
+    # *********** Check distribution list ***************
+    LSB_DIST="$(. /etc/os-release && echo "$ID")"
+    LSB_DIST="$(echo "$LSB_DIST" | tr '[:upper:]' '[:lower:]')"
+    # *********** Check distribution version ***************
+    case "$LSB_DIST" in
+        ubuntu)
+            if [ -x "$(command -v lsb_release)" ]; then
+                DIST_VERSION="$(lsb_release --codename | cut -f2)"
+            fi
+            if [ -z "$DIST_VERSION" ] && [ -r /etc/lsb-release ]; then
+                DIST_VERSION="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")"
+            fi
+            # ********* Commenting Out CDROM **********************
+            sed -i "s/\(^deb cdrom.*$\)/\#/g" /etc/apt/sources.list
+        ;;
+        debian|raspbian)
+            DIST_VERSION="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
+            case "$DIST_VERSION" in
+                9) DIST_VERSION="stretch";;
+                8) DIST_VERSION="jessie";;
+                7) DIST_VERSION="wheezy";;
+            esac
+            # ********* Commenting Out CDROM **********************
+            sed -i "s/\(^deb cdrom.*$\)/\#/g" /etc/apt/sources.list
+        ;;
+        centos)
+            if [ -z "$DIST_VERSION" ] && [ -r /etc/os-release ]; then
+                DIST_VERSION="$(. /etc/os-release && echo "$VERSION_ID")"
+            fi
+        ;;
+        rhel|ol|sles)
+            ee_notice "$LSB_DIST"
+            exit 1
+            ;;
+        *)
+            if [ -x "$(command -v lsb_release)" ]; then
+                DIST_VERSION="$(lsb_release --release | cut -f2)"
+            fi
+            if [ -z "$DIST_VERSION" ] && [ -r /etc/os-release ]; then
+                DIST_VERSION="$(. /etc/os-release && echo "$VERSION_ID")"
+            fi
+        ;;
+    esac           
+    ERROR=$?
+    if [ $ERROR -ne 0 ]; then
+        echoerror "Could not verify distribution or version of the OS (Error Code: $ERROR)."
+    fi
+    echo "[HELK-DOCKER-INSTALLATION-INFO] You're using $LSB_DIST version $DIST_VERSION" 
+elif [ "$SYSTEM_KERNEL" == "Darwin" ]; then
+    PRODUCT_NAME="$(sw_vers -productName)"
+    PRODUCT_VERSION="$(sw_vers -productVersion)"
+    BUILD_VERSION="$(sw_vers -buildVersion)"
+    echo "[HELK-DOCKER-INSTALLATION-INFO] You're using $PRODUCT_NAME version $PRODUCT_VERSION"
+else
+    echo "[HELK-DOCKER-INSTALLATION-INFO] We cannot figure out the SYSTEM_KERNEL, distribution or version of the OS"
+fi
+
 
 # ********** Install Curl ********************
 install_curl(){      
     echo "[HELK-DOCKER-INSTALLATION-INFO] Installing curl before installing docker.."
-    case "$lsb_dist" in
+    case "$LSB_DIST" in
         ubuntu|debian|raspbian)
             apt-get install -y curl >> $LOGFILE 2>&1
         ;;
         centos|rhel)
             yum install curl >> $LOGFILE 2>&1
         ;;
         *)
-            echo "[HELK-DOCKER-INSTALLATION-INFO] Please install curl for $lsb_dist $dist_version.."
+            echo "[HELK-DOCKER-INSTALLATION-INFO] Please install curl for $LSB_DIST $DIST_VERSION .."
             exit 1
         ;;
     esac
@@ -81,8 +141,8 @@ install_docker_compose(){
     fi
 }
 
-# *********** Main steps
-if [ "$systemKernel" == "Linux" ]; then
+# *********** Main steps *********************
+if [ "$SYSTEM_KERNEL" == "Linux" ]; then
     # *********** Check if curl is installed ***************
     if [ -x "$(command -v curl)" ]; then
         echo "[HELK-DOCKER-INSTALLATION-INFO] curl is already installed"
@@ -110,7 +170,7 @@ else
     if [ -x "$(command -v docker)" ] && [ -x "$(command -v docker-compose)" ]; then
         echo "[HELK-DOCKER-INSTALLATION-INFO] Docker & Docker-compose already installed"
     else
-        echo "[HELK-DOCKER-INSTALLATION-INFO] Please innstall Docker & Docker-compose for $systemKernel"
+        echo "[HELK-DOCKER-INSTALLATION-INFO] Please innstall Docker & Docker-compose for $SYSTEM_KERNEL"
         exit 1
     fi
 fi
diff --git a/docker/helk_install.sh b/docker/helk_install.sh
@@ -36,7 +36,7 @@ check_min_requirements(){
             echo "[HELK-INSTALLATION-ERROR] Installation Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/Installation"
             exit 1
         fi
-        if [ "${AVAILABLE_MEMORY}" -ge "11" ] && [ "${AVAILABLE_DISK}" -ge "25" ]; then
+        if [ "${AVAILABLE_MEMORY}" -ge "12" ] && [ "${AVAILABLE_DISK}" -ge "25" ]; then
             echo "[HELK-INSTALLATION-INFO] Available Memory: $AVAILABLE_MEMORY"
             echo "[HELK-INSTALLATION-INFO] Available Disk: $AVAILABLE_DISK"
         else
@@ -442,7 +442,7 @@ show_banner(){
     echo "**          HELK - THE HUNTING ELK          **"
     echo "**                                          **"
     echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **"
-    echo "** HELK build version: v0.1.6-alpha01312019 **"
+    echo "** HELK build version: v0.1.6-alpha02022019 **"
     echo "** HELK ELK version: 6.5.4                  **"
     echo "** License: GPL-3.0                         **"
     echo "**********************************************"
@@ -483,9 +483,9 @@ install_helk(){
     check_system_info
     set_helk_build
     set_helk_subscription
+    set_network
     set_kibana_ui_password
     set_elasticsearch_password
-    set_network
     prepare_helk
     build_helk
     sleep 180