From b2bcb904661018dacac5de28ae2a4cb5620cd28a Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Thu, 3 Oct 2024 16:13:00 +0200
Subject: [PATCH 01/10] Changed goaccess metrics origin

---
 extract_goaccess_metrics.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 6e5a4e1..c4f9ca4 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -12,11 +12,11 @@ LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log"
 mkdir -p $OSCAR_LOGS_DIR
 
 # Log format for goaccess
-LOG_FORMAT='%^ %^ %^ %h - - [%d:%t] %~ %m %U %^ %s %^ %R %^ %^'
+LOG_FORMAT='%^ %^ %^ %^ [%^] %d - %t | %s | %Tms | %h | %m %~ %U | %u'
 
 addLog(){
     ingress_logfile=$1
-    cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null
+    cat $ingress_logfile | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null
 }
 
 metrics(){
@@ -54,11 +54,11 @@ metrics(){
 
 for log in "$CLUSTER_LOGS_DIR"/*;
 do
-    if [[ $log == *"ingress"* ]]; then
+    if [[ $log == *"oscar_oscar"* ]]; then
         cp -r $log $LOCAL_LOGS_DIR
         # remove total path
         log=$(echo $log | sed 's/\/var\/log\/clusterlogs\///')
-        for logfile in "$LOCAL_LOGS_DIR/$log/controller/"*;
+        for logfile in "$LOCAL_LOGS_DIR/$log/oscar/"*;
         do
             if [[ $logfile == *".gz" ]]; then
                 # upload a backup of the ingress logs to s3
@@ -71,12 +71,12 @@ do
     fi
 done
 
-for logfile in "$LOCAL_LOGS_DIR/$log/controller/"*;
+for logfile in "$LOCAL_LOGS_DIR/$log/oscar/"*;
 do
     if [[ $logfile == *".log"* ]]; then
         if [[ $logfile == *".log" ]]; then
             aws s3 cp $logfile s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
-            cat $logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
+            cat $logfile | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
             metrics $LATEST_LOGS
         else
             addLog $logfile

From da6c65903753c0647edd153cab2647fa73d50177 Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Fri, 4 Oct 2024 09:58:41 +0200
Subject: [PATCH 02/10] Updated goaccess config file

---
 goaccess.conf | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/goaccess.conf b/goaccess.conf
index a4579af..4139929 100755
--- a/goaccess.conf
+++ b/goaccess.conf
@@ -10,7 +10,7 @@
 # The following time format works with any of the
 # Apache/NGINX's log formats below.
 #
-time-format %H:%M:%S +0000
+time-format %H:%M:%S
 #
 # Google Cloud Storage or
 # The time in microseconds since the Unix epoch.
@@ -51,7 +51,9 @@ time-format %H:%M:%S +0000
 #
 #date-format %s
 
-date-format %d/%b/%Y
+#date-format %d/%b/%Y
+
+date-format %Y/%m/%d
 
 ######################################
 # Log Format Options (required)

From 2144dcf3d18fa28bf3f0f99debf3910ea3220dfa Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Fri, 4 Oct 2024 10:50:53 +0200
Subject: [PATCH 03/10] Missing grep filter

---
 extract_goaccess_metrics.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index c4f9ca4..867b1a4 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -16,7 +16,7 @@ LOG_FORMAT='%^ %^ %^ %^ [%^] %d - %t | %s | %Tms | %h | %m %~ %U | %u'
 
 addLog(){
     ingress_logfile=$1
-    cat $ingress_logfile | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null
+    cat $ingress_logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null
 }
 
 metrics(){
@@ -76,7 +76,7 @@ do
     if [[ $logfile == *".log"* ]]; then
         if [[ $logfile == *".log" ]]; then
             aws s3 cp $logfile s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
-            cat $logfile | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
+            cat $logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
             metrics $LATEST_LOGS
         else
             addLog $logfile

From 6f2a5acab84d276995a1d41faa9122bd3db277ef Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Mon, 7 Oct 2024 12:51:29 +0200
Subject: [PATCH 04/10] Minor change on log format

---
 extract_goaccess_metrics.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 867b1a4..3f946d2 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -12,7 +12,7 @@ LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log"
 mkdir -p $OSCAR_LOGS_DIR
 
 # Log format for goaccess
-LOG_FORMAT='%^ %^ %^ %^ [%^] %d - %t | %s | %Tms | %h | %m %~ %U | %u'
+LOG_FORMAT='%^ %^ %^ %^ [%^] %d - %t | %s | %Ts | %h | %m %~ %U | %u'
 
 addLog(){
     ingress_logfile=$1

From ffafe04a1e037b1bf3486101efb4763246cea44a Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Mon, 14 Oct 2024 09:47:36 +0200
Subject: [PATCH 05/10] Changed s3 backup logs

---
 extract_goaccess_metrics.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 3f946d2..aba6937 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -56,13 +56,13 @@ for log in "$CLUSTER_LOGS_DIR"/*;
 do
     if [[ $log == *"oscar_oscar"* ]]; then
         cp -r $log $LOCAL_LOGS_DIR
+        # upload a backup of the logs to s3
+        aws s3 cp --recursive $log s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
         # remove total path
         log=$(echo $log | sed 's/\/var\/log\/clusterlogs\///')
         for logfile in "$LOCAL_LOGS_DIR/$log/oscar/"*;
         do
             if [[ $logfile == *".gz" ]]; then
-                # upload a backup of the ingress logs to s3
-                aws s3 cp $logfile s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
                 # unzip all log files
                 gzip -d $logfile
             fi
@@ -75,7 +75,6 @@ for logfile in "$LOCAL_LOGS_DIR/$log/oscar/"*;
 do
     if [[ $logfile == *".log"* ]]; then
         if [[ $logfile == *".log" ]]; then
-            aws s3 cp $logfile s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
             cat $logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
             metrics $LATEST_LOGS
         else

From db3e215f53d4351d83c527349d3c6a1fd2223eee Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Mon, 14 Oct 2024 10:12:45 +0200
Subject: [PATCH 06/10] Fixed s3 upload

---
 extract_goaccess_metrics.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index aba6937..1b71532 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -52,15 +52,15 @@ metrics(){
     done
 }
 
-for log in "$CLUSTER_LOGS_DIR"/*;
+for log_path in "$CLUSTER_LOGS_DIR"/*;
 do
-    if [[ $log == *"oscar_oscar"* ]]; then
-        cp -r $log $LOCAL_LOGS_DIR
-        # upload a backup of the logs to s3
-        aws s3 cp --recursive $log s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
+    if [[ $log_path == *"oscar_oscar"* ]]; then
+        cp -r $log_path $LOCAL_LOGS_DIR
         # remove total path
-        log=$(echo $log | sed 's/\/var\/log\/clusterlogs\///')
-        for logfile in "$LOCAL_LOGS_DIR/$log/oscar/"*;
+        log_relative_path=$(echo $log | sed 's/\/var\/log\/clusterlogs\///')
+        # upload a backup of the logs to s3
+        aws s3 cp --recursive $log_path s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/"${log_relative_path}"
+        for logfile in "$LOCAL_LOGS_DIR/$log_relative_path/oscar/"*;
         do
             if [[ $logfile == *".gz" ]]; then
                 # unzip all log files

From c33a9f6b24dc9bc465d4f6d57053a419ab03ceb7 Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Mon, 14 Oct 2024 11:09:04 +0200
Subject: [PATCH 07/10] Fixed path error

---
 extract_goaccess_metrics.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 1b71532..286b092 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -57,7 +57,7 @@ do
     if [[ $log_path == *"oscar_oscar"* ]]; then
         cp -r $log_path $LOCAL_LOGS_DIR
         # remove total path
-        log_relative_path=$(echo $log | sed 's/\/var\/log\/clusterlogs\///')
+        log_relative_path=$(echo $log_path | sed 's/\/var\/log\/clusterlogs\///')
         # upload a backup of the logs to s3
         aws s3 cp --recursive $log_path s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/"${log_relative_path}"
         for logfile in "$LOCAL_LOGS_DIR/$log_relative_path/oscar/"*;

From ed4d18391002226cd8a7e02cb7690e2c7c661b1a Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Mon, 14 Oct 2024 11:24:09 +0200
Subject: [PATCH 08/10] Fixed log_path

---
 extract_goaccess_metrics.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 286b092..6f5ec2f 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -71,7 +71,7 @@ do
     fi
 done
 
-for logfile in "$LOCAL_LOGS_DIR/$log/oscar/"*;
+for logfile in "$LOCAL_LOGS_DIR/$log_path/oscar/"*;
 do
     if [[ $logfile == *".log"* ]]; then
         if [[ $logfile == *".log" ]]; then

From 547a4de038582327fb7504ae367d2c20ebb9232e Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Tue, 15 Oct 2024 16:24:36 +0200
Subject: [PATCH 09/10] Minor fix

---
 extract_goaccess_metrics.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 6f5ec2f..7c843da 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -71,7 +71,8 @@ do
     fi
 done
 
-for logfile in "$LOCAL_LOGS_DIR/$log_path/oscar/"*;
+# /var/log/ingresslogs/oscar_oscar-7499cd/oscar
+for logfile in "$LOCAL_LOGS_DIR/$relative_log_path/oscar/"*;
 do
     if [[ $logfile == *".log"* ]]; then
         if [[ $logfile == *".log" ]]; then

From 46c5c0d22cedab22ec7f7aa476c7a1dee6844a82 Mon Sep 17 00:00:00 2001
From: catttam <caterinaalarconm@gmail.com>
Date: Wed, 16 Oct 2024 09:43:43 +0200
Subject: [PATCH 10/10] Typo fix

---
 extract_goaccess_metrics.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index 7c843da..e4ef203 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -57,7 +57,7 @@ do
     if [[ $log_path == *"oscar_oscar"* ]]; then
         cp -r $log_path $LOCAL_LOGS_DIR
         # remove total path
-        log_relative_path=$(echo $log_path | sed 's/\/var\/log\/clusterlogs\///')
+        relative_log_path=$(echo $log_path | sed 's/\/var\/log\/clusterlogs\///')
         # upload a backup of the logs to s3
         aws s3 cp --recursive $log_path s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/"${log_relative_path}"
         for logfile in "$LOCAL_LOGS_DIR/$log_relative_path/oscar/"*;