From a4782f2c2b69dec93eecfc68e04cec8228a24534 Mon Sep 17 00:00:00 2001
From: Christian Rohmann <christian.rohmann@inovex.de>
Date: Mon, 12 Aug 2024 17:42:55 +0200
Subject: [PATCH] Increase readiness and liveness probe timeouts, introduce an
 initial delay

Currently the default timeout of 1 second and no initial delay is applied
to the probes of the runner pods. Depending on the startup time this
can cause random Pod errors causing a whole TestRun to fail.

At some point it might also make sense to introduce a startupProbe to cover
the longer initial startup time a K6 instance / pod might need instead of ever
increasing the runtime liveness and readiness checks.

Since having the Liveness and Readiness checks be just the same makes not much
sense, as the liveness check fail will cause the container to be restarted,
this change also splits up those two tests, to allow for more individual
configuration, be it timers or what is actually checked.

Fixes #306

Signed-off-by: Christian Rohmann <christian.rohmann@inovex.de>
---
 pkg/resources/jobs/runner.go      | 24 +++++++++++++++++++++---
 pkg/resources/jobs/runner_test.go |  3 +++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/pkg/resources/jobs/runner.go b/pkg/resources/jobs/runner.go
index 54105401..ec2c9bb7 100644
--- a/pkg/resources/jobs/runner.go
+++ b/pkg/resources/jobs/runner.go
@@ -185,8 +185,8 @@ func NewRunnerJob(k6 v1alpha1.TestRunI, index int, token string) (*batchv1.Job,
 						VolumeMounts:    volumeMounts,
 						Ports:           ports,
 						EnvFrom:         k6.GetSpec().Runner.EnvFrom,
-						LivenessProbe:   generateProbe(k6.GetSpec().Runner.LivenessProbe),
-						ReadinessProbe:  generateProbe(k6.GetSpec().Runner.ReadinessProbe),
+						LivenessProbe:   generateLivenessProbe(k6.GetSpec().Runner.LivenessProbe),
+						ReadinessProbe:  generateReadinessProbe(k6.GetSpec().Runner.ReadinessProbe),
 						SecurityContext: &k6.GetSpec().Runner.ContainerSecurityContext,
 					}},
 					TerminationGracePeriodSeconds: &zero,
@@ -276,11 +276,29 @@ func newAntiAffinity() *corev1.Affinity {
 	}
 }
 
-func generateProbe(configuredProbe *corev1.Probe) *corev1.Probe {
+func generateLivenessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
 	if configuredProbe != nil {
 		return configuredProbe
 	}
 	return &corev1.Probe{
+		InitialDelaySeconds: 10,
+		TimeoutSeconds:      3,
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/v1/status",
+				Port:   intstr.IntOrString{IntVal: 6565},
+				Scheme: "HTTP",
+			},
+		},
+	}
+}
+
+func generateReadinessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
+	if configuredProbe != nil {
+		return configuredProbe
+	}
+	return &corev1.Probe{
+		TimeoutSeconds: 3,
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
 				Path:   "/v1/status",
diff --git a/pkg/resources/jobs/runner_test.go b/pkg/resources/jobs/runner_test.go
index 22078cf5..472b3f85 100644
--- a/pkg/resources/jobs/runner_test.go
+++ b/pkg/resources/jobs/runner_test.go
@@ -339,6 +339,8 @@ func TestNewRunnerJob(t *testing.T) {
 							},
 						},
 						LivenessProbe: &corev1.Probe{
+							InitialDelaySeconds: 10,
+							TimeoutSeconds:      3,
 							ProbeHandler: corev1.ProbeHandler{
 								HTTPGet: &corev1.HTTPGetAction{
 									Path:   "/v1/status",
@@ -348,6 +350,7 @@ func TestNewRunnerJob(t *testing.T) {
 							},
 						},
 						ReadinessProbe: &corev1.Probe{
+							TimeoutSeconds: 3,
 							ProbeHandler: corev1.ProbeHandler{
 								HTTPGet: &corev1.HTTPGetAction{
 									Path:   "/v1/status",