[Feature] : Halt testing upon the first failure

crds/testsuite-json-schema.yaml

@@ -43,6 +43,10 @@ properties:
     description: If set, do not delete the mocked control plane or kind cluster.
     type: boolean
     default: false
+  stopOnFirstFailure:
+    description: StopOnFirstFailure determines whether the test should stop upon encountering the first failure.
+    type: boolean
+    default: false   
   timeout:
     description: Override the default timeout of 30 seconds (in seconds).
     type: integer

crds/testsuite_crd.yaml

@@ -59,6 +59,10 @@ spec:
               description: If set, do not delete the mocked control plane or kind cluster.
               type: boolean
               default: false
+            stopOnFirstFailure:
+              description: StopOnFirstFailure determines whether the test should stop upon encountering the first failure.
+              type: boolean
+              default: false
             timeout:
               description: Override the default timeout of 30 seconds (in seconds).
               type: integer

pkg/apis/testharness/v1beta1/test_types.go

@@ -51,6 +51,8 @@ type TestSuite struct {
 	SkipDelete bool `json:"skipDelete"`
 	// If set, do not delete the mocked control plane or kind cluster.
 	SkipClusterDelete bool `json:"skipClusterDelete"`
+	// StopOnFirstFailure determines whether the test should stop upon encountering the first failure.
+	StopOnFirstFailure bool `json:"stopOnFirstFailure"`
 	// Override the default timeout of 30 seconds (in seconds).
 	// +kubebuilder:validation:Format:=int64
 	Timeout int `json:"timeout"`

pkg/kuttlctl/cmd/test.go

@@ -52,6 +52,7 @@ func newTestCmd() *cobra.Command { //nolint:gocyclo
 	kindContext := ""
 	skipDelete := false
 	skipClusterDelete := false
+	stopOnFirstFailure := false
 	parallel := 0
 	artifactsDir := ""
 	mockControllerFile := ""
@@ -161,6 +162,10 @@ For more detailed documentation, visit: https://kuttl.dev`,
 				options.SkipClusterDelete = skipClusterDelete
 			}
 
+			if isSet(flags, "stop-on-first-failure") {
+				options.StopOnFirstFailure = stopOnFirstFailure
+			}
+
 			if isSet(flags, "parallel") {
 				options.Parallel = parallel
 			}
@@ -253,6 +258,7 @@ For more detailed documentation, visit: https://kuttl.dev`,
 	testCmd.Flags().StringVar(&artifactsDir, "artifacts-dir", "", "Directory to output kind logs to (if not specified, the current working directory).")
 	testCmd.Flags().BoolVar(&skipDelete, "skip-delete", false, "If set, do not delete resources created during tests (helpful for debugging test failures, implies --skip-cluster-delete).")
 	testCmd.Flags().BoolVar(&skipClusterDelete, "skip-cluster-delete", false, "If set, do not delete the mocked control plane or kind cluster.")
+	testCmd.Flags().BoolVar(&stopOnFirstFailure, "stop-on-first-failure", false, "Stop testing upon the first failure encountered")
 	// The default value here is only used for the help message. The default is actually enforced in RunTests.
 	testCmd.Flags().IntVar(&parallel, "parallel", 8, "The maximum number of tests to run at once.")
 	testCmd.Flags().IntVar(&timeout, "timeout", 30, "The timeout to use as default for TestSuite configuration.")

pkg/test/harness.go

@@ -399,6 +399,8 @@ func (h *Harness) RunTests() {
 		}
 	}
 
+	failureOccurred := false
+
 	h.T.Run("harness", func(t *testing.T) {
 		for testDir, tests := range realTestSuite {
 			h.T.Logf("testsuite: %s has %d tests", testDir, len(tests))
@@ -428,8 +430,23 @@ func (h *Harness) RunTests() {
 
 					tc := report.NewCase(name)
 					test.Run(t, tc)
+					if tc.Failure != nil {
+						// assuming tc.Failure is set when a test case fails
+						failureOccurred = true
+					}
 					suite.AddTestcase(tc)
+					// Check after every test case if a failure has occurred
+					if failureOccurred && h.TestSuite.StopOnFirstFailure {
+						t.SkipNow()
+						return
+					}
 				})
+				if failureOccurred && h.TestSuite.StopOnFirstFailure {
+					break
+				}
+			}
+			if failureOccurred && h.TestSuite.StopOnFirstFailure {
+				break
 			}
 		}
 	})