feat: metric integration for fsm (#2155)

backend/controller/controller.go

@@ -7,6 +7,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/observability"
 	"hash"
 	"io"
 	"math/rand"
@@ -226,6 +227,10 @@ func New(ctx context.Context, pool *pgxpool.Pool, config Config, runnerScaling s
 	}
 	config.SetDefaults()
 
+	if err := observability.InitControllerObservability(); err != nil {
+		log.FromContext(ctx).Warnf("failed to initialize controller observability: %v", err)
+	}
+
 	// Override some defaults during development mode.
 	_, devel := runnerScaling.(*localscaling.LocalScaling)
 	if devel {

backend/controller/dal/fsm.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/observability"
 	"time"
 
 	"github.com/alecthomas/types/optional"
@@ -57,6 +58,7 @@ func (d *DAL) StartFSMTransition(ctx context.Context, fsm schema.RefKey, executi
 		}
 		return fmt.Errorf("failed to start FSM transition: %w", err)
 	}
+	observability.FSMInstanceCreated(ctx, fsm)
 	return nil
 }
 
@@ -67,11 +69,13 @@ func (d *DAL) FinishFSMTransition(ctx context.Context, fsm schema.RefKey, instan
 
 func (d *DAL) FailFSMInstance(ctx context.Context, fsm schema.RefKey, instanceKey string) error {
 	_, err := d.db.FailFSMInstance(ctx, fsm, instanceKey)
+	observability.FSMInstanceCompleted(ctx, fsm)
 	return dalerrs.TranslatePGError(err)
 }
 
 func (d *DAL) SucceedFSMInstance(ctx context.Context, fsm schema.RefKey, instanceKey string) error {
 	_, err := d.db.SucceedFSMInstance(ctx, fsm, instanceKey)
+	observability.FSMInstanceCompleted(ctx, fsm)
 	return dalerrs.TranslatePGError(err)
 }
 

backend/controller/observability/fsm.go

@@ -0,0 +1,52 @@
+package observability
+
+import (
+	"context"
+	"fmt"
+	"github.com/TBD54566975/ftl/backend/schema"
+	"github.com/TBD54566975/ftl/internal/observability"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	fsmMeterName    = "ftl.fsm"
+	fsmRefAttribute = "ftl.fsm.ref"
+)
+
+var fsmMeter = otel.Meter("ftl.fsm")
+
+var fsmCounters = struct {
+	instancesActive metric.Int64UpDownCounter
+}{}
+
+func InitFSMMetrics() error {
+	var err error
+
+	fsmCounters.instancesActive, err = fsmMeter.Int64UpDownCounter(
+		fmt.Sprintf("%s.instances.active", fsmMeterName),
+		metric.WithDescription("counts the number of active FSM instances"))
+
+	if err != nil {
+		return fmt.Errorf("could not initialize fsm metrics: %w", err)
+	}
+
+	return nil
+}
+
+func FSMInstanceCreated(ctx context.Context, fsm schema.RefKey) {
+	if fsmCounters.instancesActive != nil {
+		fsmCounters.instancesActive.Add(ctx, 1, metric.WithAttributes(
+			attribute.String(observability.ModuleNameAttribute, fsm.Module),
+			attribute.String(fsmRefAttribute, fsm.String())))
+	}
+}
+
+func FSMInstanceCompleted(ctx context.Context, fsm schema.RefKey) {
+	if fsmCounters.instancesActive != nil {
+		fsmCounters.instancesActive.Add(ctx, -1, metric.WithAttributes(
+			attribute.String(observability.ModuleNameAttribute, fsm.Module),
+			attribute.String(fsmRefAttribute, fsm.String())))
+	}
+}

backend/controller/observability/observability.go

@@ -0,0 +1,11 @@
+package observability
+
+import "fmt"
+
+func InitControllerObservability() error {
+	if err := InitFSMMetrics(); err != nil {
+		return fmt.Errorf("could not initialize controller metrics: %w", err)
+	}
+
+	return nil
+}

internal/observability/attributes.go

@@ -0,0 +1,5 @@
+package observability
+
+const (
+	ModuleNameAttribute = "ftl.module.name"
+)