fix: fixed inconsistent and disappearing pod statuses, added failed e… (

#28) * fix: fixed inconsistent and disappearing pod statuses, added failed events info and cleaned up inconsistencies * fix:github action test version update to use go 1.19 * fix:upgrade kustomize to latest to support go 1.19 version * refactor: crd is reformatted due to kustomize version upgrade * refactor: remove commented line Co-authored-by: Sangeetha Madamanchi <[email protected]>
ExpediaGroup · Nov 17, 2022 · 98eba51 · 98eba51
1 parent 2b6e9f5
commit 98eba51
Show file tree

Hide file tree

Showing 19 changed files with 899 additions and 614 deletions.
diff --git a/.github/workflows/publish-latest.yaml b/.github/workflows/publish-latest.yaml
@@ -5,7 +5,7 @@ on:
     branches: [master]
     types: [completed]
 env:
-  VERSION: v0.0.2
+  VERSION: v0.0.3
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -24,7 +24,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.19
       - uses: actions/checkout@v3
       - name: Log in to the Container registry
         uses: docker/[email protected]

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -16,7 +16,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.19
       - uses: actions/checkout@v3
       - name: Build binary to make sure it works
         run: go build

diff --git a/Makefile b/Makefile
@@ -177,7 +177,7 @@ controller-gen: ## Download controller-gen locally if necessary.
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 .PHONY: kustomize
 kustomize: ## Download kustomize locally if necessary.
-	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
+	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@latest)
 
 ENVTEST = $(shell pwd)/bin/setup-envtest
 .PHONY: envtest
@@ -203,7 +203,7 @@ TMP_DIR=$$(mktemp -d) ;\
 cd $$TMP_DIR ;\
 go mod init tmp ;\
 echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
 endef

diff --git a/README.md b/README.md
@@ -37,21 +37,10 @@ To clean the Kind cluster, execute `make kind-clean`.
 ## Prerequisite
 This project is based on golang 1.17. Make sure your GoRoot is configured to 1.17. If any other version then you may face issues.
 
-## Debugging
-Since this project is a Kubernetes operator triggered by resource creation, debugging can be done using a remote debugger.
-
-### Steps to achieve remote debugging:
-
-* Make sure you install `kind`
-* Create a kind cluster with the command in your terminal, `kind create cluster --name overwhelm`
-* Use this cluster for all your debugging needs
-* To build and deploy your operator run, `make deploy`
-* Once the operator resources are deployed and running successfully in the cluster, run the `make run-delve` command
-* In GoLand use remote debugger by selecting the `GO Remote` configuration. Within the configuration, host is `localhost` and port is `2345`
-* Now you can debug your code. To test the operator logic, deploy whatever resources you need from the config/samples folder.
-* Run `make undeploy` to remove your kubernetes resources once you are done
 
 ## Testing
 * Kubebuilder uses @Ginkgo and @Gomega test suites which are configured in controllers/suite_test.go
 * Run `make test` to test your changes. 
 * Reference materials: [Gomega](https://onsi.github.io/gomega/) and [Ginkgo](https://onsi.github.io/ginkgo/)
+
+
diff --git a/analyzer/pod.go b/analyzer/pod.go
@@ -41,13 +41,13 @@ func analyzeContainers(containerStatuses []v1.ContainerStatus, result *Result) {
 	for _, container := range containerStatuses {
 		if !container.Ready {
 			result.Healthy = false
-		}
-		if container.State.Waiting != nil {
-			result.Errors = append(result.Errors, formatContainerError(container.Name, "waiting", container.State.Waiting.Reason, container.State.Waiting.Message, 0, container.Ready))
-		} else if container.State.Terminated != nil && container.State.Terminated.Reason != "Completed" {
-			result.Errors = append(result.Errors, formatContainerError(container.Name, "terminated", container.State.Terminated.Reason, container.State.Terminated.Message, container.State.Terminated.ExitCode, container.Ready))
-		} else if container.State.Running != nil && !container.Ready {
-			result.Errors = append(result.Errors, formatContainerError(container.Name, "running", "", "", 0, container.Ready))
+			if container.State.Waiting != nil {
+				result.Errors = append(result.Errors, formatContainerError(container.Name, "waiting", container.State.Waiting.Reason, container.State.Waiting.Message, 0, container.Ready))
+			} else if container.State.Terminated != nil && container.State.Terminated.Reason != "Completed" {
+				result.Errors = append(result.Errors, formatContainerError(container.Name, "terminated", container.State.Terminated.Reason, container.State.Terminated.Message, container.State.Terminated.ExitCode, container.Ready))
+			} else if container.State.Running != nil && !container.Ready {
+				result.Errors = append(result.Errors, formatContainerError(container.Name, "running", "", "", 0, container.Ready))
+			}
 		}
 	}
 }
@@ -59,20 +59,19 @@ func formatContainerError(containerName, state, reason, message string, exitCode
 	}
 	if !ready {
 		sb.WriteString(" is not ready and")
-	} else {
-		sb.WriteString(" is ready and")
-	}
-	if len(state) > 0 {
-		sb.WriteString(fmt.Sprintf(" is in a %s state", state))
-	}
-	if len(reason) > 0 {
-		sb.WriteString(fmt.Sprintf(" due to reason '%s'", reason))
-	}
-	if len(message) > 0 {
-		sb.WriteString(fmt.Sprintf(" with message '%s'", message))
-	}
-	if exitCode > 0 {
-		sb.WriteString(fmt.Sprintf(" (exit code %d)", exitCode))
+		if len(state) > 0 {
+			sb.WriteString(fmt.Sprintf(" is in a %s state", state))
+		}
+		if len(reason) > 0 {
+			sb.WriteString(fmt.Sprintf(" due to reason '%s'", reason))
+		}
+		if len(message) > 0 {
+			sb.WriteString(fmt.Sprintf(" with message '%s'", message))
+		}
+		if exitCode > 0 {
+			sb.WriteString(fmt.Sprintf(" (exit code %d)", exitCode))
+		}
 	}
 	return strings.TrimSpace(sb.String())
+
 }
diff --git a/api/v1alpha2/application_types.go b/api/v1alpha2/application_types.go
@@ -16,6 +16,7 @@ package v1alpha2
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/ExpediaGroup/overwhelm/analyzer"
 	"github.com/fluxcd/helm-controller/api/v2beta1"
@@ -24,9 +25,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+const PodReady = "PodReady"
+const ContainersNotReady = "ContainersNotReady"
+const PodInitializing = "PodInitializing"
+
 type Metadata struct {
 	Labels      map[string]string `json:"labels,omitempty"`
 	Annotations map[string]string `json:"annotations,omitempty"`
@@ -139,18 +143,20 @@ func AppErrorStatus(application *Application, error string) {
 	apimeta.SetStatusCondition(&application.Status.Conditions, condition)
 }
 
-func AppPodAnalysisCondition(application *Application, result analyzer.Result) {
+func AppPodAnalysisCondition(application *Application, result analyzer.Result) bool {
+	appPodReadyCondition := apimeta.FindStatusCondition(application.Status.Conditions, PodReady)
 	condition := metav1.Condition{
-		Type:   "PodReady", // Could be meta.ReadyCondition, but it would clash with the HR
-		Reason: "ContainersReady",
+		Type: PodReady, // Could be meta.ReadyCondition, but it would clash with the HR
 	}
-	if result.Healthy {
-		condition.Status = metav1.ConditionTrue
-		condition.Message = "At least one new pod has progressed successfully"
-	} else {
-		condition.Reason = "ContainersNotReady"
-		condition.Status = metav1.ConditionFalse
+	if !result.Healthy {
+
 		condition.Message = fmt.Sprintf("%s %s is unhealthy: %v", result.ResourceType, result.ResourceName, result.Errors)
+		if appPodReadyCondition == nil || strings.Contains(appPodReadyCondition.Message, PodInitializing) || !strings.Contains(condition.Message, PodInitializing) {
+			condition.Reason = ContainersNotReady
+			condition.Status = metav1.ConditionFalse
+			apimeta.SetStatusCondition(&application.Status.Conditions, condition)
+			return true
+		}
 	}
-	apimeta.SetStatusCondition(&application.Status.Conditions, condition)
+	return false
 }
diff --git a/api/v1alpha2/groupversion_info.go b/api/v1alpha2/groupversion_info.go
@@ -13,9 +13,9 @@
 // limitations under the License.
 
 // Package v1alpha2 contains API Schema definitions for the core v1alpha2 API group
-//+kubebuilder:object:generate=true
-//+groupName=core.expediagroup.com
-//+versionName:=v1alpha2
+// +kubebuilder:object:generate=true
+// +groupName=core.expediagroup.com
+// +versionName:=v1alpha2
 package v1alpha2
 
 import (

diff --git a/api/v1alpha2/reference_types.go b/api/v1alpha2/reference_types.go
@@ -14,7 +14,6 @@
 
 package v1alpha2
 
-
 type PreRenderer struct {
 	// Custom non white-spaced and non alpha-numeric open delimiter used for go templating action to pre-render. For e.g., <%. Default is {{
 	// +kubebuilder:validation:MinLength=2

diff --git a/charts/overwhelm/Chart.yaml b/charts/overwhelm/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: overwhelm
-version: 1.1.3
+version: 1.1.4
 maintainers:
     - name: "Expedia Group"
       url: "https://github.com/ExpediaGroup/overwhelm"