Fix#376 - E2E tests are failing in nightly deploy job (#384)

.ci/jenkins/Jenkinsfile.e2e.cluster

@@ -2,9 +2,12 @@
 
 helper = null
 
-minikubeClusterPlatform = 'minikube'
+KIND_VERSION = "v0.20.0"
+kindClusterPlatform = 'kind'
 openshiftClusterPlatform = 'openshift'
 
+kindLogsFolder = "/tmp/kind-logs"
+
 pipeline {
     agent {
         docker { 
@@ -47,6 +50,31 @@ pipeline {
                 }
             }
         }
+        stage('Load image into Kind') {
+            when {
+                expression { 
+                    return getClusterName() == kindClusterPlatform 
+                }
+            }
+            steps {
+                script {
+                    kind.loadImage(getTestImage())
+                }
+            }
+        }
+        stage('Deploy the operator') {
+            when {
+                expression { 
+                    return getClusterName() == kindClusterPlatform 
+                }
+            }
+            steps {
+                script {
+                    sh "make deploy IMG=${getTestImage()}"
+                    sh "kubectl wait pod -A -l control-plane=sonataflow-operator --for condition=Ready --timeout=120s"
+                }
+            }
+        }
         stage('Prepare for e2e tests') {
             when {
                 expression { 
@@ -83,16 +111,25 @@ pipeline {
                                 make test-e2e
                             """
                         } catch (err) {
+                            kind.exportLogs(kindLogsFolder)
                             sh 'make undeploy'
+                            deleteKindCluster()
                             throw err
                         }
                         sh 'kubectl get pods -A'
+                        deleteKindCluster()
                     }
                 }
             }
         }
     }
     post {
+        always {
+            script {
+                archiveArtifacts(artifacts: "**${kindLogsFolder}/**/*.*,**/e2e-test-report.xml")
+                junit '**/e2e-test-report.xml'
+            }
+        }
         cleanup {
             script {
                 clean()
@@ -104,7 +141,6 @@ pipeline {
 void clean() {
     helper.cleanGoPath()
     util.cleanNode(containerEngine)
-    cleanupCluster()
 }
 
 String getTestImage() {
@@ -121,24 +157,25 @@ String getOperatorVersion() {
 
 void setupCluster() {
     switch (getClusterName()) {
-        case minikubeClusterPlatform:
-            setupMinikube()
+        case kindClusterPlatform:
+            echo 'Creating kind cluster'
+            createKindCluster()
             break
         case openshiftClusterPlatform:
+            echo 'Setting up Openshift'
             setupOpenshift()
             break
         default:
             error "Unknown cluster name ${getClusterName()}. Cannot prepare for it ..."
     }
 }
 
-void setupMinikube() {
-    // Start minikube
-    minikube.minikubeMemory = '12g'
-    minikube.start()
+void createKindCluster() {
+    sh(script: "make KIND_VERSION=${KIND_VERSION} create-cluster", returnStdout: true)
+}
 
-    minikube.waitForMinikubeStarted()
-    minikube.waitForMinikubeRegistry()
+void deleteKindCluster() {
+    sh(script: "make delete-cluster", returnStdout: true)
 }
 
 void setupOpenshift() {
@@ -148,8 +185,9 @@ void setupOpenshift() {
 
 void cleanupCluster() {
     switch (getClusterName()) {
-        case minikubeClusterPlatform:
-            minikube.stop()
+        case kindClusterPlatform:
+            echo 'Deleting kind cluster'
+            deleteKindCluster()
             break
         case openshiftClusterPlatform:
             echo 'Nothing to cleanup on openshift. All good !'
@@ -161,8 +199,8 @@ void cleanupCluster() {
 
 void executeInCluster(Closure executeClosure) {
     switch (getClusterName()) {
-        case minikubeClusterPlatform:
-            echo "Execute in minikube"
+        case kindClusterPlatform:
+            echo "Execute in kind"
             executeClosure()
             break
         case openshiftClusterPlatform:
@@ -178,7 +216,7 @@ void executeInCluster(Closure executeClosure) {
 
 void getPlatformCRFilePath() {
     switch (getClusterName()) {
-        case minikubeClusterPlatform:
+        case kindClusterPlatform:
             return 'test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withCache_minikube.yaml'
         case openshiftClusterPlatform:
             return 'test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_openshift.yaml'

.ci/jenkins/scripts/helper.groovy

@@ -1,7 +1,7 @@
 openshift = null
 container = null
 properties = null
-minikube = null
+kind = null
 
 defaultImageParamsPrefix = 'IMAGE'
 baseImageParamsPrefix = 'BASE_IMAGE'
@@ -19,7 +19,7 @@ void initPipeline() {
     container.containerEngineTlsOptions = env.CONTAINER_ENGINE_TLS_OPTIONS ?: ''
     container.containerOpenshift = openshift
 
-    minikube = load '.ci/jenkins/scripts/minikube.groovy'
+    kind = load '.ci/jenkins/scripts/kind.groovy'
 }
 
 void updateDisplayName() {

.ci/jenkins/scripts/kind.groovy

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+void exportLogs(String destination) {
+     println "Exporting kind logs to ${destination}"
+    def exportKindLogs = sh(returnStatus: true, script: """ 
+        mkdir -p .${destination}
+        kind export logs --loglevel=debug .${destination}
+    """)
+}
+
+void loadImage(String imageName) {
+    sh "kind load docker-image ${imageName}"
+}
+
+return this

Makefile

@@ -347,7 +347,7 @@ generate-all: generate generate-deploy bundle addheaders vet fmt
 
 .PHONY: test-e2e # You will need to have a Minikube/Kind cluster up in running to run this target, and run container-builder before the test
 test-e2e: install-operator-sdk
-	go test ./test/e2e/* -v -ginkgo.v -timeout 30m
+	go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report.xml -timeout 60m
 
 .PHONY: before-pr
 before-pr: test generate-all

test/e2e/helpers.go

@@ -84,6 +84,7 @@ func verifyHealthStatusInPod(name string, namespace string) {
 			Expect(h.Status).To(Equal(upStatus))
 			return
 		}
+
 		if len(errs.Error()) > 0 {
 			errs = fmt.Errorf("%v; %w", err, errs)
 		} else {
@@ -99,7 +100,17 @@ func getHealthStatusInContainer(podName string, containerName string, ns string)
 	cmd := exec.Command("kubectl", "exec", "-t", podName, "-n", ns, "-c", containerName, "--", "curl", "-s", "localhost:8080/q/health")
 	output, err := utils.Run(cmd)
 	Expect(err).NotTo(HaveOccurred())
-	err = json.Unmarshal(output, &h)
+	// On Apache CI Nodes, does not return valid JSON, hence we match first and last brackets by index and extract it
+	stringOutput := string(output)
+	startIndex := strings.Index(stringOutput, "{")
+	endIndex := strings.LastIndex(stringOutput, "}")
+	if startIndex == 0 {
+		stringOutput = stringOutput[startIndex : endIndex+1]
+	} else {
+		stringOutput = stringOutput[startIndex-1 : endIndex+1]
+	}
+	fmt.Printf("Parsed following JSON object from health Endpoint response: %v\n", stringOutput)
+	err = json.Unmarshal([]byte(stringOutput), &h)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute curl command against health endpoint in container %s:%v with output %s", containerName, err, output)
 	}

test/e2e/platform_test.go

@@ -87,7 +87,7 @@ var _ = Describe("Validate the persistence", Ordered, func() {
 				cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app=sonataflow-platform", "--for", "condition=Ready", "--timeout=5s")
 				_, err = utils.Run(cmd)
 				return err
-			}, 10*time.Minute, 5).Should(Succeed())
+			}, 20*time.Minute, 5).Should(Succeed())
 			By("Evaluate status of service's health endpoint")
 			cmd = exec.Command("kubectl", "get", "pod", "-l", "app=sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}")
 			output, err := utils.Run(cmd)
@@ -113,7 +113,7 @@ var _ = Describe("Validate the persistence", Ordered, func() {
 				Expect(sf).NotTo(BeEmpty(), "sonataflow name is empty")
 				EventuallyWithOffset(1, func() bool {
 					return verifyWorkflowIsInRunningStateInNamespace(sf, targetNamespace)
-				}, 5*time.Minute, 5).Should(BeTrue())
+				}, 10*time.Minute, 5).Should(BeTrue())
 			}
 		},
 			Entry("with both Job Service and Data Index and ephemeral persistence and the workflow in a dev profile", test.GetSonataFlowE2EPlatformServicesDirectory(), dev, ephemeral),

test/e2e/workflow_test.go

@@ -68,7 +68,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
 					"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 
 			By("check the workflow is in running state")
 			EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("simple", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())
@@ -78,7 +78,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
 					"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 		})
 
 		It("should successfully deploy the Greeting Workflow in prod mode and verify if it's running", func() {
@@ -88,15 +88,15 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
 					"test/testdata/"+test.SonataFlowGreetingsDataInputSchemaConfig), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 
 			By("creating an instance of the SonataFlow Operand(CR)")
 			EventuallyWithOffset(1, func() error {
 				cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir,
 					"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 
 			By("check the workflow is in running state")
 			EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("greeting", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())
@@ -106,7 +106,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
 					"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 		})
 
 		It("should successfully deploy the orderprocessing workflow in devmode and verify if it's running", func() {
@@ -117,7 +117,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
 					test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 
 			By("check the workflow is in running state")
 			EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("orderprocessing", targetNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue())
@@ -135,7 +135,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
 					test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace)
 				_, err := utils.Run(cmd)
 				return err
-			}, 2*time.Minute, time.Second).Should(Succeed())
+			}, 3*time.Minute, time.Second).Should(Succeed())
 		})
 
 	})
@@ -184,7 +184,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() {
 			out, err := utils.Run(cmd)
 			GinkgoWriter.Printf("%s\n", string(out))
 			return err
-		}, 10*time.Minute, 5).Should(Succeed())
+		}, 12*time.Minute, 5).Should(Succeed())
 
 		By("Evaluate status of the workflow's pod database connection health endpoint")
 		cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app=callbackstatetimeouts", "-n", ns, "-ojsonpath={.items[*].metadata.name}")
@@ -205,7 +205,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() {
 				}
 			}
 			return false
-		}, 10*time.Minute).Should(BeTrue())
+		}, 12*time.Minute).Should(BeTrue())
 	},
 		Entry("defined in the workflow from an existing kubernetes service as a reference", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("by_service")),
 	)