From c9349d4c59d5632b7e0b50bd7a0b98014003997c Mon Sep 17 00:00:00 2001 From: vimystic <122659254+vimystic@users.noreply.github.com> Date: Tue, 16 Apr 2024 20:26:41 -0600 Subject: [PATCH] Add logic to see if pod is stuck for more than a minute --- internal/fullnode/stuck_detection.go | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/internal/fullnode/stuck_detection.go b/internal/fullnode/stuck_detection.go index 0fc727b0..a75de0ec 100644 --- a/internal/fullnode/stuck_detection.go +++ b/internal/fullnode/stuck_detection.go @@ -58,7 +58,38 @@ func (d StuckPodDetection) StuckPods(ctx context.Context, crd *cosmosv1.CosmosFu } func isPodStuck(receivedString string) bool { - return strings.Contains(receivedString, "SignerListener: Connected") + if strings.Contains(receivedString, "SignerListener: Connected") { + timeInLog, err := extractTimeFromLog(receivedString) + if err != nil { + fmt.Println("Error parsing time from log:", err) + return true + } + + currentTime := time.Now().UTC() + + logTimeToday := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), + timeInLog.Hour(), timeInLog.Minute(), timeInLog.Second(), timeInLog.Nanosecond(), currentTime.Location()) + + timeDiff := currentTime.Sub(logTimeToday) + + if timeDiff >= time.Minute { + return true + } + } + + return false +} + +func extractTimeFromLog(log string) (time.Time, error) { + parts := strings.Fields(log) + + const timeLayout = "3:04PM" + parsedTime, err := time.Parse(timeLayout, parts[0]) + if err != nil { + return time.Time{}, err + } + + return parsedTime, nil } func getPodLogsLastLine(clientset *kubernetes.Clientset, pod *corev1.Pod) string {