From cbc4118ea7d9992204f081d12b01ac3c39be3707 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:16:09 -0700
Subject: [PATCH] feat: metric integration for fsm (#2155)

---
 backend/controller/controller.go              |  5 ++
 backend/controller/dal/fsm.go                 |  4 ++
 backend/controller/observability/fsm.go       | 52 +++++++++++++++++++
 .../controller/observability/observability.go | 11 ++++
 internal/observability/attributes.go          |  5 ++
 internal/observability/metrics/attributes.go  | 18 -------
 6 files changed, 77 insertions(+), 18 deletions(-)
 create mode 100644 backend/controller/observability/fsm.go
 create mode 100644 backend/controller/observability/observability.go
 create mode 100644 internal/observability/attributes.go
 delete mode 100644 internal/observability/metrics/attributes.go

diff --git a/backend/controller/controller.go b/backend/controller/controller.go
index f05f7e72a7..662069abbe 100644
--- a/backend/controller/controller.go
+++ b/backend/controller/controller.go
@@ -7,6 +7,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/observability"
 	"hash"
 	"io"
 	"math/rand"
@@ -226,6 +227,10 @@ func New(ctx context.Context, pool *pgxpool.Pool, config Config, runnerScaling s
 	}
 	config.SetDefaults()
 
+	if err := observability.InitControllerObservability(); err != nil {
+		log.FromContext(ctx).Warnf("failed to initialize controller observability: %v", err)
+	}
+
 	// Override some defaults during development mode.
 	_, devel := runnerScaling.(*localscaling.LocalScaling)
 	if devel {
diff --git a/backend/controller/dal/fsm.go b/backend/controller/dal/fsm.go
index a88f2c42ec..d8a5a3855a 100644
--- a/backend/controller/dal/fsm.go
+++ b/backend/controller/dal/fsm.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/observability"
 	"time"
 
 	"github.com/alecthomas/types/optional"
@@ -57,6 +58,7 @@ func (d *DAL) StartFSMTransition(ctx context.Context, fsm schema.RefKey, executi
 		}
 		return fmt.Errorf("failed to start FSM transition: %w", err)
 	}
+	observability.FSMInstanceCreated(ctx, fsm)
 	return nil
 }
 
@@ -67,11 +69,13 @@ func (d *DAL) FinishFSMTransition(ctx context.Context, fsm schema.RefKey, instan
 
 func (d *DAL) FailFSMInstance(ctx context.Context, fsm schema.RefKey, instanceKey string) error {
 	_, err := d.db.FailFSMInstance(ctx, fsm, instanceKey)
+	observability.FSMInstanceCompleted(ctx, fsm)
 	return dalerrs.TranslatePGError(err)
 }
 
 func (d *DAL) SucceedFSMInstance(ctx context.Context, fsm schema.RefKey, instanceKey string) error {
 	_, err := d.db.SucceedFSMInstance(ctx, fsm, instanceKey)
+	observability.FSMInstanceCompleted(ctx, fsm)
 	return dalerrs.TranslatePGError(err)
 }
 
diff --git a/backend/controller/observability/fsm.go b/backend/controller/observability/fsm.go
new file mode 100644
index 0000000000..64da7e7935
--- /dev/null
+++ b/backend/controller/observability/fsm.go
@@ -0,0 +1,52 @@
+package observability
+
+import (
+	"context"
+	"fmt"
+	"github.com/TBD54566975/ftl/backend/schema"
+	"github.com/TBD54566975/ftl/internal/observability"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	fsmMeterName    = "ftl.fsm"
+	fsmRefAttribute = "ftl.fsm.ref"
+)
+
+var fsmMeter = otel.Meter("ftl.fsm")
+
+var fsmCounters = struct {
+	instancesActive metric.Int64UpDownCounter
+}{}
+
+func InitFSMMetrics() error {
+	var err error
+
+	fsmCounters.instancesActive, err = fsmMeter.Int64UpDownCounter(
+		fmt.Sprintf("%s.instances.active", fsmMeterName),
+		metric.WithDescription("counts the number of active FSM instances"))
+
+	if err != nil {
+		return fmt.Errorf("could not initialize fsm metrics: %w", err)
+	}
+
+	return nil
+}
+
+func FSMInstanceCreated(ctx context.Context, fsm schema.RefKey) {
+	if fsmCounters.instancesActive != nil {
+		fsmCounters.instancesActive.Add(ctx, 1, metric.WithAttributes(
+			attribute.String(observability.ModuleNameAttribute, fsm.Module),
+			attribute.String(fsmRefAttribute, fsm.String())))
+	}
+}
+
+func FSMInstanceCompleted(ctx context.Context, fsm schema.RefKey) {
+	if fsmCounters.instancesActive != nil {
+		fsmCounters.instancesActive.Add(ctx, -1, metric.WithAttributes(
+			attribute.String(observability.ModuleNameAttribute, fsm.Module),
+			attribute.String(fsmRefAttribute, fsm.String())))
+	}
+}
diff --git a/backend/controller/observability/observability.go b/backend/controller/observability/observability.go
new file mode 100644
index 0000000000..8d84df13f0
--- /dev/null
+++ b/backend/controller/observability/observability.go
@@ -0,0 +1,11 @@
+package observability
+
+import "fmt"
+
+func InitControllerObservability() error {
+	if err := InitFSMMetrics(); err != nil {
+		return fmt.Errorf("could not initialize controller metrics: %w", err)
+	}
+
+	return nil
+}
diff --git a/internal/observability/attributes.go b/internal/observability/attributes.go
new file mode 100644
index 0000000000..ff6bd6c558
--- /dev/null
+++ b/internal/observability/attributes.go
@@ -0,0 +1,5 @@
+package observability
+
+const (
+	ModuleNameAttribute = "ftl.module.name"
+)
diff --git a/internal/observability/metrics/attributes.go b/internal/observability/metrics/attributes.go
deleted file mode 100644
index b254d35870..0000000000
--- a/internal/observability/metrics/attributes.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package metrics
-
-import (
-	"github.com/TBD54566975/ftl/backend/schema"
-	"go.opentelemetry.io/otel/attribute"
-)
-
-// ModuleNameAttribute identifies the name of the module that the associated
-// metric originates from.
-func ModuleNameAttribute(name string) attribute.KeyValue {
-	return attribute.String("ftl.module.name", name)
-}
-
-// VerbRefAttribute identifies the verb that the associated metric originates
-// from. The entire module qualified name is used: e.g. {module.verb}
-func VerbRefAttribute(ref schema.Ref) attribute.KeyValue {
-	return attribute.String("ftl.verb.ref", ref.Name)
-}