diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh index 53bff5f..d1fcae9 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_goaccess_metrics.sh @@ -7,7 +7,8 @@ CLUSTER_LOGS_DIR="/var/log/clusterlogs" LOCAL_LOGS_DIR="/var/log/ingresslogs" OSCAR_LOGS_DIR="$LOCAL_LOGS_DIR/oscar" -FULL_REPORT_FILE="$OSCAR_LOGS_DIR/oscar.log" +FULL_LOGS="$OSCAR_LOGS_DIR/oscar.log" +LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log" mkdir -p $OSCAR_LOGS_DIR # Log format for goaccess @@ -15,25 +16,25 @@ LOG_FORMAT='%^ %^ %^ %h - - [%d:%t] %~ %m %U %^ %s %^ %R %^ %^' addLog(){ ingress_logfile=$1 - cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $FULL_REPORT_FILE >/dev/null + cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $FULL_LOGS >/dev/null } metrics(){ - - geo_err=$( { goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_full.json" --json-pretty-print; } 2>&1 ) - python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_full.json" -g 0 + LOG_FILE=$1 + geo_err=$( { goaccess "${LOG_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${LOG_FILE}_full.json" --json-pretty-print; } 2>&1 ) + python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${LOG_FILE}_full.json" -g 0 status_codes=('200' '204' '404' '500') init="t" - out="${FILTERED_PATH}/${OUTPUT_FILE}" + out="${FILTERED_PATH}/${LOG_FILE}" for code in "${status_codes[@]}"; do - code_logs=$(cat $FULL_REPORT_FILE| grep -e 'HTTP/[0-9].[0-9]" '${code}' ') + code_logs=$(cat $LOG_FILE| grep -e 'HTTP/[0-9].[0-9]" '${code}' ') if [ ! -z "$code_logs" ]; then - app_err=$( { cat $FULL_REPORT_FILE | grep -e 'HTTP/[0-9].[0-9]" '${code}' ' | goaccess - -o "${out}_f${code}.json" --json-pretty-print --log-format="${LOG_FORMAT}"; } 2>&1 ) + app_err=$( { cat $LOG_FILE | grep -e 'HTTP/[0-9].[0-9]" '${code}' ' | goaccess - -o "${out}_f${code}.json" --json-pretty-print --log-format="${LOG_FORMAT}"; } 2>&1 ) if [ ! -f "${out}_f${code}.json" ]; then - echo "[*] Warning: Couldn't process file $FULL_REPORT_FILE for status code '$code'" + echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'" else if [ $init == 't' ]; then python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code @@ -67,15 +68,23 @@ done for logfile in "$LOCAL_LOGS_DIR/$log/controller/"*; do - if [[ $logfile == *".log"* && $logfile != *".log" ]]; then - addLog $logfile + if [[ $logfile == *".log"* ]]; then + if [[ $logfile == *".log"]]; then + cat $logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null + metrics $LATEST_LOGS + else + addLog $logfile + fi fi done # Generate the html file -if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then +if [ ! -f "${FULL_LOGS}" ] || [ ! -s "${FULL_LOGS}" ]; then echo "Error: Failed to create html report." exit 1 fi -metrics -goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" \ No newline at end of file + +metrics $FULL_LOGS + +cat $LATEST_LOGS | tee -a $FULL_LOGS >/dev/null +goaccess "${FULL_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" \ No newline at end of file diff --git a/goaccess_metric_parser.py b/goaccess_metric_parser.py index b3682a2..997b116 100644 --- a/goaccess_metric_parser.py +++ b/goaccess_metric_parser.py @@ -27,8 +27,13 @@ with open(args.file_path, 'r') as rawfile: metrics = json.loads(rawfile.read()) - START_DATE = metrics["general"]["start_date"] - END_DATE = metrics["general"]["end_date"] + try: + START_DATE = metrics["general"]["start_date"] + END_DATE = metrics["general"]["end_date"] + except: + START_DATE = metrics["general"]["date_time"] + END_DATE = metrics["general"]["date_time"] + """ > Countries reached diff --git a/metrics_prom.py b/metrics_prom.py index 01c79da..c3f1264 100644 --- a/metrics_prom.py +++ b/metrics_prom.py @@ -7,9 +7,9 @@ from oscar_python.client import Client QUERY_ENDPOINT = "/api/v1/query?query=" -TIME = "30d" +TIME = "5d" END_DATE = datetime.today() -START_DATE = END_DATE - timedelta(days=30) +START_DATE = END_DATE - timedelta(days=5) parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)