metrics: added statsd-exporter and simple prometheus rules

Aiven-Open · Aug 23, 2024 · 7f5a4b7 · 7f5a4b7
1 parent 44b578f
commit 7f5a4b7
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 2 deletions.
diff --git a/container/compose.yml b/container/compose.yml
@@ -78,6 +78,8 @@ services:
       KARAPACE_TOPIC_NAME: _schemas
       KARAPACE_LOG_LEVEL: WARNING
       KARAPACE_COMPATIBILITY: FULL
+      KARAPACE_STATSD_HOST: statsd-exporter
+      KARAPACE_STATSD_PORT: 8125
 
   karapace-rest:
     image: ghcr.io/aiven-open/karapace:develop
@@ -102,10 +104,32 @@ services:
       KARAPACE_REGISTRY_PORT: 8081
       KARAPACE_ADMIN_METADATA_MAX_AGE: 0
       KARAPACE_LOG_LEVEL: WARNING
+      KARAPACE_STATSD_HOST: statsd-exporter
+      KARAPACE_STATSD_PORT: 8125
 
   prometheus:
     image: prom/prometheus
     volumes:
-    - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    - ./prometheus/rules.yml:/etc/prometheus/rules.yml
     ports:
       - 9090:9090
+
+  grafana:
+      image: grafana/grafana
+      environment:
+          GF_SECURITY_ADMIN_USER: karapace
+          GF_SECURITY_ADMIN_PASSWORD: karapace
+          GF_PATHS_PROVISIONING: /grafana/provisioning
+      ports:
+          - 3000:3000
+      volumes:
+          - ./grafana/dashboards:/grafana/dashboards
+          - ./grafana/provisioning:/grafana/provisioning
+
+  statsd-exporter:
+      image: prom/statsd-exporter
+      command: "--statsd.listen-udp=:8125 --web.listen-address=:9102"
+      ports:
+        - 9102:9102
+        - 8125:8125/udp
diff --git a/container/prometheus.yml → container/prometheus/prometheus.yml b/container/prometheus.yml → container/prometheus/prometheus.yml
@@ -1,7 +1,10 @@
 global:
   scrape_interval: 10s # How frequently to scrape targets by default.
   scrape_timeout: 5s # How long until a scrape request times out.
-  evaluation_interval: 60s # How frequently to evaluate rules.
+  evaluation_interval: 10s # How frequently to evaluate rules.
+
+rule_files:
+ - /etc/prometheus/rules.yml
 
 # A scrape configuration
 scrape_configs:
@@ -16,3 +19,11 @@ scrape_configs:
     static_configs:
       - targets:
         - karapace-rest:8082
+
+  - job_name: statsd-exporter
+    metrics_path: /metrics
+    tls_config:
+      insecure_skip_verify: true
+    static_configs:
+      - targets:
+        - statsd-exporter:9102
diff --git a/container/prometheus/rules.yml b/container/prometheus/rules.yml
@@ -0,0 +1,20 @@
+groups:
+  - name: karapace
+    rules:
+    - record: karapace_exceptions_sum_by_exception
+      expr: sum by (exception) (exception)
+    - alert: HighHTTPRequests
+      expr: karapace_http_requests_total > 10
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: High HTTP requests for (instance={{ $labels.instance }})
+        description: "Service received\n  HTTP Requests = {{ $value }}\n"
+    - alert: FireImmidiately
+      expr: karapace_schema_reader_schemas > 1
+      labels:
+        severity: page
+      annotations:
+        summary: Lots of schems on (instance={{ $labels.instance }})
+        description: "\n Schema count = {{ $value }}\n"