Changed goaccess parser

extract_goaccess_metrics.sh

@@ -7,33 +7,34 @@ CLUSTER_LOGS_DIR="/var/log/clusterlogs"
 LOCAL_LOGS_DIR="/var/log/ingresslogs"
 OSCAR_LOGS_DIR="$LOCAL_LOGS_DIR/oscar"
 
-FULL_REPORT_FILE="$OSCAR_LOGS_DIR/oscar.log"
+FULL_LOGS="$OSCAR_LOGS_DIR/oscar.log"
+LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log"
 mkdir -p $OSCAR_LOGS_DIR
 
 # Log format for goaccess
 LOG_FORMAT='%^ %^ %^ %h - - [%d:%t] %~ %m %U %^ %s %^ %R %^ %^'
 
 addLog(){
     ingress_logfile=$1
-    cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $FULL_REPORT_FILE >/dev/null
+    cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $FULL_LOGS >/dev/null
 }
 
 metrics(){
-
-    geo_err=$( { goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_full.json" --json-pretty-print; } 2>&1 )
-    python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_full.json" -g 0
+    LOG_FILE=$1
+    geo_err=$( { goaccess "${LOG_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${LOG_FILE}_full.json" --json-pretty-print; } 2>&1 )
+    python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${LOG_FILE}_full.json" -g 0
 
     status_codes=('200' '204' '404' '500')
     init="t"
 
-    out="${FILTERED_PATH}/${OUTPUT_FILE}"
+    out="${FILTERED_PATH}/${LOG_FILE}"
 
     for code in "${status_codes[@]}"; do
-    code_logs=$(cat $FULL_REPORT_FILE| grep -e 'HTTP/[0-9].[0-9]" '${code}' ')
+    code_logs=$(cat $LOG_FILE| grep -e 'HTTP/[0-9].[0-9]" '${code}' ')
     if [ ! -z "$code_logs" ]; then
-        app_err=$( { cat $FULL_REPORT_FILE | grep -e 'HTTP/[0-9].[0-9]" '${code}' ' | goaccess - -o  "${out}_f${code}.json" --json-pretty-print --log-format="${LOG_FORMAT}"; } 2>&1 )
+        app_err=$( { cat $LOG_FILE | grep -e 'HTTP/[0-9].[0-9]" '${code}' ' | goaccess - -o  "${out}_f${code}.json" --json-pretty-print --log-format="${LOG_FORMAT}"; } 2>&1 )
         if [ ! -f "${out}_f${code}.json" ]; then
-            echo "[*] Warning: Couldn't process file $FULL_REPORT_FILE for status code '$code'"
+            echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'"
         else
             if [ $init == 't' ]; then
                 python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code
@@ -67,15 +68,23 @@ done
 
 for logfile in "$LOCAL_LOGS_DIR/$log/controller/"*;
 do
-    if [[ $logfile == *".log"* && $logfile != *".log" ]]; then
-    addLog $logfile
+    if [[ $logfile == *".log"* ]]; then
+        if [[ $logfile == *".log"]]; then
+            cat $logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
+            metrics $LATEST_LOGS
+        else
+            addLog $logfile
+        fi
     fi
 done
 
 # Generate the html file
-if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then
+if [ ! -f "${FULL_LOGS}" ] || [ ! -s "${FULL_LOGS}" ]; then
     echo "Error: Failed to create html report."
     exit 1
 fi
-metrics
-goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
+
+metrics $FULL_LOGS
+
+cat $LATEST_LOGS | tee -a $FULL_LOGS >/dev/null
+goaccess "${FULL_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"

goaccess_metric_parser.py

@@ -27,8 +27,13 @@
 
 with open(args.file_path, 'r') as rawfile:
     metrics = json.loads(rawfile.read())
-    START_DATE = metrics["general"]["start_date"]
-    END_DATE = metrics["general"]["end_date"]
+    try:
+        START_DATE = metrics["general"]["start_date"]
+        END_DATE = metrics["general"]["end_date"]
+    except:
+        START_DATE = metrics["general"]["date_time"]
+        END_DATE = metrics["general"]["date_time"]
+
 
 """
  > Countries reached 

metrics_prom.py

@@ -7,9 +7,9 @@
 from oscar_python.client import Client
 
 QUERY_ENDPOINT = "/api/v1/query?query="
-TIME = "30d"
+TIME = "5d"
 END_DATE = datetime.today()
-START_DATE = END_DATE - timedelta(days=30)
+START_DATE = END_DATE - timedelta(days=5)
 
 parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)