Skip to content

Commit

Permalink
Add logic to see if pod is stuck for more than a minute
Browse files Browse the repository at this point in the history
  • Loading branch information
vimystic committed Apr 17, 2024
1 parent 5b1d880 commit c9349d4
Showing 1 changed file with 32 additions and 1 deletion.
33 changes: 32 additions & 1 deletion internal/fullnode/stuck_detection.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,38 @@ func (d StuckPodDetection) StuckPods(ctx context.Context, crd *cosmosv1.CosmosFu
}

func isPodStuck(receivedString string) bool {
return strings.Contains(receivedString, "SignerListener: Connected")
if strings.Contains(receivedString, "SignerListener: Connected") {
timeInLog, err := extractTimeFromLog(receivedString)
if err != nil {
fmt.Println("Error parsing time from log:", err)
return true
}

currentTime := time.Now().UTC()

logTimeToday := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(),
timeInLog.Hour(), timeInLog.Minute(), timeInLog.Second(), timeInLog.Nanosecond(), currentTime.Location())

timeDiff := currentTime.Sub(logTimeToday)

if timeDiff >= time.Minute {
return true
}
}

return false
}

func extractTimeFromLog(log string) (time.Time, error) {
parts := strings.Fields(log)

const timeLayout = "3:04PM"
parsedTime, err := time.Parse(timeLayout, parts[0])
if err != nil {
return time.Time{}, err
}

return parsedTime, nil
}

func getPodLogsLastLine(clientset *kubernetes.Clientset, pod *corev1.Pod) string {
Expand Down

0 comments on commit c9349d4

Please sign in to comment.