feat: adopt Grafana for metrics visualization (#2364) (#2378)

fixes: #2364 

Introduces a Grafana / OTEL container for capturing, ephemerally
storing, and visualizing OTEL metrics emitted by FTL. The collection
frequency is also increased to once per second (from a default once per
minute)

<img width="1246" alt="Screenshot 2024-08-14 at 7 58 39 PM"
src="https://github.com/user-attachments/assets/bd21cd33-dc93-4a79-9b2a-ad124f2db296">

Justfile

@@ -194,27 +194,6 @@ debug *args:
   dlv_pid=$!
   wait "$dlv_pid"
 
-# otel is short for OpenTelemetry.
-otelGrpcPort := `cat docker-compose.yml | grep "OTLP gRPC" | sed 's/:.*//' | sed -r 's/ +- //'`
-
-# Run otel collector behind a webapp to stream local (i.e. from ftl dev) signals to your
-# browser. Ctrl+C to stop. To start FTL, open another terminal tab and run `just otel-dev`
-# with any args you would pass to `ftl dev`.
-#
-# CAUTION: The version of otel-desktop-viewer we are running with only has support for
-# traces, NOT metrics or logs. If you want to view metrics or logs, you will need to use
-# `just otel-stream` below. The latest version of otel-desktop-viewer has support for
-# metrics and logs, but it is not available to be installed yet. Refer to issue:
-#     https://github.com/CtrlSpice/otel-desktop-viewer/issues/146
-otel-ui:
-  #!/bin/bash
-
-  if ! test -f $(git rev-parse --show-toplevel)/.hermit/go/bin/otel-desktop-viewer ; then
-    echo "Installing otel-desktop-viewer..."
-    go install github.com/CtrlSpice/otel-desktop-viewer@latest
-  fi
-  otel-desktop-viewer --grpc {{otelGrpcPort}}
-
 # Run otel collector in a docker container to stream local (i.e. from ftl dev) signals to
 # the terminal tab where this is running. To start FTL, opepn another terminal tab and run
 # `just otel-dev` with any args you would pass to `ftl dev`. To stop the otel stream, run
@@ -223,26 +202,38 @@ otel-stream:
   #!/bin/bash
 
   docker run \
-    -p {{otelGrpcPort}}:{{otelGrpcPort}} \
+    -p ${OTEL_GRPC_PORT}:${OTEL_GRPC_PORT} \
     -p 55679:55679 \
     otel/opentelemetry-collector:0.104.0 2>&1 | sed 's/\([A-Z].* #\)/\
   \1/g'
 
 # Stop the docker container running otel.
-otelContainerID := `docker ps -f ancestor=otel/opentelemetry-collector:0.104.0 | tail -1 | cut -d " " -f1`
+otel-container-id := `docker ps -f ancestor=otel/opentelemetry-collector:0.104.0 | tail -1 | cut -d " " -f1`
+
 otel-stop:
-  docker stop "{{otelContainerID}}"
+  docker stop "{{otel-container-id}}"
 
 # Run `ftl dev` with the given args after setting the necessary envar.
 otel-dev *args:
   #!/bin/bash
 
-  grpcPort=$(cat docker-compose.yml | grep "OTLP gRPC" | sed 's/:.*//' | sed -r 's/ +- //')
-  export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${grpcPort}"
+  export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${OTEL_GRPC_PORT}"
+  export OTEL_METRIC_EXPORT_INTERVAL=${OTEL_METRIC_EXPORT_INTERVAL}
   # Uncomment this line for much richer debug logs
   # export FTL_O11Y_LOG_LEVEL="debug"
   ftl dev {{args}}
 
+# Runs a Grafana stack for storing and visualizing telemetry. This stack includes a
+# Prometheus database for metrics and a Tempo database for traces; both of which are
+# populated by the OTLP over GRPC collector that is integrated with this stack.
+#
+# Running `just otel-dev` will export ftl metrics to this Grafana stack.
+grafana:
+  docker compose up -d grafana
+
+grafana-stop:
+  docker compose down grafana
+
 storybook:
   #!/bin/bash
-  cd frontend && npm run storybook
+  cd frontend && npm run storybook

bin/hermit.hcl

@@ -3,6 +3,8 @@ env = {
   "FTL_ENDPOINT": "http://localhost:8892",
   "FTL_INIT_GO_REPLACE": "github.com/TBD54566975/ftl=${HERMIT_ENV}",
   "FTL_SOURCE": "${HERMIT_ENV}",
+  "OTEL_GRPC_PORT": "4317",
+  "OTEL_HTTP_PORT": "4318",
   "OTEL_METRIC_EXPORT_INTERVAL": "5000",
   "PATH": "${HERMIT_ENV}/scripts:${HERMIT_ENV}/frontend/node_modules/.bin:${HERMIT_ENV}/extensions/vscode/node_modules/.bin:${PATH}",
 }

docker-compose.yml

@@ -18,6 +18,22 @@ services:
       timeout: 60s
       retries: 60
       start_period: 80s
+  grafana:
+    profiles:
+      - infra
+    image: grafana/otel-lgtm
+    platform: linux/amd64
+    restart: unless-stopped
+    stdin_open: true # docker run -i
+    tty: true        # docker run -t
+    ports:
+      - 3000:3000 # Portal Endpoint
+      - ${OTEL_GRPC_PORT}:4317 # OTLP GRPC Collector
+      - ${OTEL_HTTP_PORT}:4318 # OTLP HTTP Collector
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - grafana-storage:/var/lib/grafana
   otel-collector:
     profiles:
       - infra
@@ -36,3 +52,6 @@ services:
     environment:
       SERVICES: secretsmanager
       DEBUG: 1
+
+volumes:
+  grafana-storage: {}