From b6c4ab29c9fd66afaf2a3e9e4a4e3960dc4c398c Mon Sep 17 00:00:00 2001 From: Nikolina Gyurova Date: Mon, 13 Nov 2023 14:07:44 +0200 Subject: [PATCH 1/3] feat: Ignore specific errors for given pods --- controller.go | 5 +++++ helm/templates/deployment.yaml | 2 ++ helm/values.yaml | 4 ++++ helpers.go | 41 ++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/controller.go b/controller.go index f9404a7..acbc51a 100644 --- a/controller.go +++ b/controller.go @@ -263,6 +263,11 @@ func (c *Controller) handlePod(pod *v1.Pod) error { if err != nil { return err } + + if isIgnoredErrorForPod(pod.Name, lastNonEmptyLogLine(containerLogs)) { + continue + } + if containerLogs == "" { containerLogs = "• No Logs Before Restart\n" } else { diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 6c179c9..8f57b2d 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -49,6 +49,8 @@ spec: value: {{ .Values.watchedPodNamePrefixes | quote}} - name: IGNORED_POD_NAME_PREFIXES value: {{ .Values.ignoredPodNamePrefixes | quote}} + - name: IGNORED_ERROR_FOR_POD_NAME_PREFIXES + value: {{ .Values.ignoredErrorsForPodNamePrefixes | quote}} - name: IGNORE_RESTARTS_WITH_EXIT_CODE_ZERO value: {{ .Values.ignoreRestartsWithExitCodeZero | quote}} - name: SLACK_WEBHOOK_URL diff --git a/helm/values.yaml b/helm/values.yaml index d1caa6c..56fb9ae 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -18,6 +18,10 @@ ignoredNamespaces: "" # A comma-separated list of pod name prefixes to ignore ignoredPodNamePrefixes: "" +# A json map with specific errors to ignore from specific pods +# {"podName": ["error 1", "error 2"]} +ignoredErrorsForPodNamePrefixes: "" + # A comma-separated list of namespaces to watch, default is all ("") watchedNamespaces: "" # A comma-separated list of pod name prefixes to watch, default is all (""). diff --git a/helpers.go b/helpers.go index ae57b1d..8a8aefa 100644 --- a/helpers.go +++ b/helpers.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "encoding/json" "os" "sort" "strconv" @@ -57,6 +58,46 @@ func isIgnoredPod(name string) bool { return false } +func isIgnoredErrorForPod(podName string, errorLog string) bool { + ignoredErrorsForPodNamePrefixesEnv := os.Getenv("IGNORED_ERRORS_FOR_POD_NAME_PREFIXES") + if ignoredErrorsForPodNamePrefixesEnv == "" { + return false + } + + podErrorsMap := make(map[string][]interface{}) + err := json.Unmarshal([]byte(ignoredErrorsForPodNamePrefixesEnv), &podErrorsMap) + if err != nil { + klog.Infof("Failed to load IGNORED_ERRORS_FOR_POD_NAME_PREFIXES with error: %s", err) + return false + } + + for key, errors := range podErrorsMap { + if strings.HasPrefix(podName, key) { + for _, ignoredError := range errors { + if strings.Contains(errorLog, ignoredError.(string)) { + klog.Infof("Ignore: pod %s has ignored error: %s\n", podName, ignoredError) + return true + } + } + } + } + + return false +} + +func lastNonEmptyLogLine(logs string) string { + logLines := strings.Split(logs, "\n") + + for i := 1; i <= len(logLines); i++ { + lastLogLine := logLines[len(logLines)-i] + if lastLogLine != "" { + return lastLogLine; + } + } + + return "" +} + func isWatchedNamespace(namespace string) bool { watchedNamespacesEnv := os.Getenv("WATCHED_NAMESPACES") if watchedNamespacesEnv == "" { From 17cecdd76f22c43ec8b4fb167822ba3ccd9baf58 Mon Sep 17 00:00:00 2001 From: Nikolina Gyurova Date: Thu, 16 Nov 2023 13:21:55 +0200 Subject: [PATCH 2/3] Fix env var name --- helm/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 8f57b2d..f8ad9eb 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -49,7 +49,7 @@ spec: value: {{ .Values.watchedPodNamePrefixes | quote}} - name: IGNORED_POD_NAME_PREFIXES value: {{ .Values.ignoredPodNamePrefixes | quote}} - - name: IGNORED_ERROR_FOR_POD_NAME_PREFIXES + - name: IGNORED_ERRORS_FOR_POD_NAME_PREFIXES value: {{ .Values.ignoredErrorsForPodNamePrefixes | quote}} - name: IGNORE_RESTARTS_WITH_EXIT_CODE_ZERO value: {{ .Values.ignoreRestartsWithExitCodeZero | quote}} From 76c605e508bbb065c79fc3e71cdc1b6d65e69caf Mon Sep 17 00:00:00 2001 From: Nikolina Gyurova Date: Fri, 29 Dec 2023 15:04:05 +0200 Subject: [PATCH 3/3] Search in all logs, not only last line --- controller.go | 2 +- helpers.go | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/controller.go b/controller.go index acbc51a..5db2668 100644 --- a/controller.go +++ b/controller.go @@ -264,7 +264,7 @@ func (c *Controller) handlePod(pod *v1.Pod) error { return err } - if isIgnoredErrorForPod(pod.Name, lastNonEmptyLogLine(containerLogs)) { + if isIgnoredErrorForPod(pod.Name, containerLogs) { continue } diff --git a/helpers.go b/helpers.go index dccf491..8f144d9 100644 --- a/helpers.go +++ b/helpers.go @@ -88,19 +88,6 @@ func isIgnoredErrorForPod(podName string, errorLog string) bool { return false } -func lastNonEmptyLogLine(logs string) string { - logLines := strings.Split(logs, "\n") - - for i := 1; i <= len(logLines); i++ { - lastLogLine := logLines[len(logLines)-i] - if lastLogLine != "" { - return lastLogLine; - } - } - - return "" -} - func isWatchedNamespace(namespace string) bool { watchedNamespacesEnv := os.Getenv("WATCHED_NAMESPACES") if watchedNamespacesEnv == "" {