From fe429b841869cc1c1070a071e0f77f3b03d8b3b4 Mon Sep 17 00:00:00 2001
From: Safeer Jiwan <safeer@tbd.email>
Date: Fri, 23 Aug 2024 14:44:51 -0700
Subject: [PATCH] fix panic in observability call

---
 backend/controller/controller.go                | 6 +++++-
 backend/controller/cronjobs/dal/dal.go          | 3 ++-
 backend/controller/observability/async_calls.go | 5 +++++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/backend/controller/controller.go b/backend/controller/controller.go
index 1bf70292a9..5d46e73d87 100644
--- a/backend/controller/controller.go
+++ b/backend/controller/controller.go
@@ -1405,7 +1405,11 @@ func (s *Service) executeAsyncCalls(ctx context.Context) (interval time.Duration
 		logger.Tracef("No async calls to execute")
 		return time.Second * 2, nil
 	} else if err != nil {
-		observability.AsyncCalls.Acquired(ctx, call.Verb, call.CatchVerb, call.Origin.String(), call.ScheduledAt, call.Catching, err)
+		if call == nil {
+			observability.AsyncCalls.AcquireFailed(ctx, err)
+		} else {
+			observability.AsyncCalls.Acquired(ctx, call.Verb, call.CatchVerb, call.Origin.String(), call.ScheduledAt, call.Catching, err)
+		}
 		return 0, err
 	}
 
diff --git a/backend/controller/cronjobs/dal/dal.go b/backend/controller/cronjobs/dal/dal.go
index 236ac13965..e75d278155 100644
--- a/backend/controller/cronjobs/dal/dal.go
+++ b/backend/controller/cronjobs/dal/dal.go
@@ -5,13 +5,14 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/alecthomas/types/optional"
+
 	"github.com/TBD54566975/ftl/backend/controller/cronjobs/sql"
 	"github.com/TBD54566975/ftl/backend/controller/observability"
 	dalerrs "github.com/TBD54566975/ftl/backend/dal"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/internal/model"
 	"github.com/TBD54566975/ftl/internal/slices"
-	"github.com/alecthomas/types/optional"
 )
 
 type DAL struct {
diff --git a/backend/controller/observability/async_calls.go b/backend/controller/observability/async_calls.go
index 471cc557c4..e37e51fc73 100644
--- a/backend/controller/observability/async_calls.go
+++ b/backend/controller/observability/async_calls.go
@@ -107,6 +107,11 @@ func (m *AsyncCallMetrics) Acquired(ctx context.Context, verb schema.RefKey, cat
 	m.acquired.Add(ctx, 1, metric.WithAttributes(attrs...))
 }
 
+// AcquireFailed should be called if an acquisition failed before any call data could be retrieved.
+func (m *AsyncCallMetrics) AcquireFailed(ctx context.Context, err error) {
+	m.acquired.Add(ctx, 1, metric.WithAttributes(observability.SuccessOrFailureStatusAttr(false)))
+}
+
 func (m *AsyncCallMetrics) Executed(ctx context.Context, verb schema.RefKey, catchVerb optional.Option[schema.RefKey], origin string, scheduledAt time.Time, isCatching bool, maybeFailureMode optional.Option[string]) {
 	attrs := extractAsyncCallAttrs(verb, catchVerb, origin, scheduledAt, isCatching)