feat: OTel metrics metering

container/compose.yml

@@ -78,7 +78,7 @@ services:
       KARAPACE_GROUP_ID: karapace-schema-registry
       KARAPACE_MASTER_ELIGIBILITY: true
       KARAPACE_TOPIC_NAME: _schemas
-      KARAPACE_LOG_LEVEL: DEBUG
+      KARAPACE_LOG_LEVEL: INFO
       KARAPACE_COMPATIBILITY: FULL
       KARAPACE_STATSD_HOST: statsd-exporter
       KARAPACE_STATSD_PORT: 8125
@@ -117,7 +117,7 @@ services:
       KARAPACE_REGISTRY_HOST: karapace-schema-registry
       KARAPACE_REGISTRY_PORT: 8081
       KARAPACE_ADMIN_METADATA_MAX_AGE: 0
-      KARAPACE_LOG_LEVEL: DEBUG
+      KARAPACE_LOG_LEVEL: INFO
       KARAPACE_STATSD_HOST: statsd-exporter
       KARAPACE_STATSD_PORT: 8125
       KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
@@ -156,6 +156,11 @@ services:
 
   prometheus:
     image: prom/prometheus
+    command:
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=1d
+      - --enable-feature=otlp-write-receiver
+      - --config.file=/etc/prometheus/prometheus.yml
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - ./prometheus/rules.yml:/etc/prometheus/rules.yml

container/opentelemetry/collector-config.yaml

@@ -15,7 +15,7 @@ exporters:
     tls:
       insecure: true
   otlphttp/prometheus:
-    endpoint: prometheus:9090/api/v1/otlp
+    endpoint: http://prometheus:9090/api/v1/otlp
     tls:
       insecure: true
 

container/prometheus/prometheus.yml

@@ -3,6 +3,10 @@ global:
   scrape_timeout: 5s # How long until a scrape request times out.
   evaluation_interval: 10s # How frequently to evaluate rules.
 
+storage:
+  tsdb:
+    out_of_order_time_window: 30m
+
 rule_files:
  - /etc/prometheus/rules.yml
 

src/karapace/config.py

@@ -29,6 +29,7 @@ class KarapaceTags(BaseModel):
 
 class KarapaceTelemetry(BaseModel):
     otel_endpoint_url: str | None = None
+    metrics_export_interval_milliseconds: int = 10000
     resource_service_name: str = "karapace"
     resource_service_instance_id: str = "karapace"
     resource_telemetry_sdk_name: str = "opentelemetry"

src/schema_registry/telemetry/meter.py

@@ -0,0 +1,32 @@
+"""
+Copyright (c) 2024 Aiven Ltd
+See LICENSE for details
+"""
+
+from dependency_injector.wiring import inject, Provide
+from karapace.config import Config
+from karapace.container import KarapaceContainer
+from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, MetricReader, PeriodicExportingMetricReader
+from typing import Final
+
+
+class Meter:
+    START_TIME_KEY: Final = "start_time"
+
+    @staticmethod
+    @inject
+    def get_meter(config: Config = Provide[KarapaceContainer.config]) -> metrics.Meter:
+        return metrics.get_meter_provider().get_meter(f"{config.tags.app}.meter")
+
+    @staticmethod
+    @inject
+    def get_metric_reader(config: Config = Provide[KarapaceContainer.config]) -> MetricReader:
+        exporter = ConsoleMetricExporter()
+        if config.telemetry.otel_endpoint_url:
+            exporter = OTLPMetricExporter(endpoint=config.telemetry.otel_endpoint_url)
+        return PeriodicExportingMetricReader(
+            exporter=exporter,
+            export_interval_millis=config.telemetry.metrics_export_interval_milliseconds,
+        )

tests/conftest.py

@@ -194,6 +194,7 @@ def fixture_karapace_container() -> KarapaceContainer:
         modules=[
             schema_registry.controller,
             schema_registry.telemetry.tracer,
+            schema_registry.telemetry.meter,
         ]
     )
     return karapace_container

tests/unit/schema_registry/telemetry/test_meter.py

@@ -0,0 +1,45 @@
+"""
+schema_registry - telemetry meter tests
+
+Copyright (c) 2024 Aiven Ltd
+See LICENSE for details
+"""
+
+from karapace.container import KarapaceContainer
+from schema_registry.telemetry.meter import Meter
+from unittest.mock import patch
+
+
+def test_meter(karapace_container: KarapaceContainer):
+    with patch("schema_registry.telemetry.meter.metrics") as mock_metrics:
+        Meter.get_meter(config=karapace_container.config())
+        mock_metrics.get_meter_provider.return_value.get_meter.assert_called_once_with("Karapace.meter")
+
+
+def test_get_metric_reader_with_otel_endpoint(karapace_container: KarapaceContainer) -> None:
+    with (
+        patch("schema_registry.telemetry.meter.OTLPMetricExporter") as mock_otlp_exporter,
+        patch("schema_registry.telemetry.meter.PeriodicExportingMetricReader") as mock_periodic_exporting_metric_reader,
+    ):
+        karapace_container.config().telemetry.otel_endpoint_url = "http://otel:4317"
+        reader = Meter.get_metric_reader(config=karapace_container.config())
+        mock_otlp_exporter.assert_called_once_with(endpoint="http://otel:4317")
+        mock_periodic_exporting_metric_reader.assert_called_once_with(
+            exporter=mock_otlp_exporter.return_value,
+            export_interval_millis=10000,
+        )
+        assert reader is mock_periodic_exporting_metric_reader.return_value
+
+
+def test_get_metric_reader_without_otel_endpoint(karapace_container: KarapaceContainer) -> None:
+    with (
+        patch("schema_registry.telemetry.meter.ConsoleMetricExporter") as mock_console_exporter,
+        patch("schema_registry.telemetry.meter.PeriodicExportingMetricReader") as mock_periodic_exporting_metric_reader,
+    ):
+        reader = Meter.get_metric_reader(config=karapace_container.config())
+        mock_console_exporter.assert_called_once()
+        mock_periodic_exporting_metric_reader.assert_called_once_with(
+            exporter=mock_console_exporter.return_value,
+            export_interval_millis=10000,
+        )
+        assert reader is mock_periodic_exporting_metric_reader.return_value