cloud, plz: add events on failed creation with simple timeout

grafana · Sep 19, 2023 · d343649 · d343649
1 parent b001713
commit d343649
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 2 deletions.
diff --git a/controllers/k6_create.go b/controllers/k6_create.go
@@ -58,6 +58,12 @@ func CreateJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Reco
 	log.Info("Creating test jobs")
 
 	if res, err = createJobSpecs(ctx, log, k6, r, token); err != nil {
+
+		if k6.IsTrue(v1alpha1.CloudTestRun) {
+			events := cloud.ErrorEvent(cloud.K6OperatorStartError).WithDetail(fmt.Sprintf("Failed to create runner jobs: %v", err))
+			cloud.SendTestRunEvents(r.k6CloudClient, k6.Spec.TestRunID, log, events)
+		}
+
 		return res, err
 	}
 

diff --git a/controllers/k6_start.go b/controllers/k6_start.go
@@ -2,12 +2,14 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/grafana/k6-operator/api/v1alpha1"
+	"github.com/grafana/k6-operator/pkg/cloud"
 	"github.com/grafana/k6-operator/pkg/resources/jobs"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -62,6 +64,20 @@ func StartJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Recon
 	log.Info(fmt.Sprintf("%d/%d runner pods ready", count, k6.Spec.Parallelism))
 
 	if count != int(k6.Spec.Parallelism) {
+
+		if t, ok := k6.LastUpdate(v1alpha1.TestRunRunning); !ok {
+			// this should never happen
+			return res, errors.New("Cannot find condition TestRunRunning")
+		} else {
+			// let's try this approach
+			if time.Since(t).Minutes() > 5 {
+				if k6.IsTrue(v1alpha1.CloudTestRun) {
+					events := cloud.ErrorEvent(cloud.K6OperatorStartError).WithDetail("Creation of runner pods takes too long: perhaps, something is off with your configuration. Check if runner jobs and pods were created successfully.")
+					cloud.SendTestRunEvents(r.k6CloudClient, k6.Spec.TestRunID, log, events)
+				}
+			}
+		}
+
 		return res, nil
 	}
 

diff --git a/pkg/cloud/test_runs.go b/pkg/cloud/test_runs.go
@@ -128,9 +128,9 @@ func SendTestRunEvents(client *cloudapi.Client, refID string, log logr.Logger, e
 		return
 	}
 
-	url := fmt.Sprintf("%s/orchestrator/v1/testruns/%s/events", client.BaseURL(), refID)
-
+	url := fmt.Sprintf("%s/orchestrator/v1/testruns/%s/events", strings.TrimSuffix(client.BaseURL(), "/v1"), refID)
 	req, err := client.NewRequest("POST", url, events)
+
 	if err != nil {
 		log.Error(err, fmt.Sprintf("Failed to create events HTTP request %+v", events))
 		return