From 1ccc436cee2b467da2d7af26495a632a35a5c3cc Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Sun, 17 Mar 2024 14:42:56 +0000 Subject: [PATCH] Add grafana dashboard for monitoring queues Fixes a bug in the worker where a single worker would take all available work!!! Adds a docker-compose.yml for using Grafana + a dashboard that streams the queue length from Redis. Signed-off-by: Dave Tucker --- dashboards/instruct-lab-bot.json | 231 +++++++++++++++++++++++ docker-compose.yml | 23 +++ provisioning/dashboards/dashboard.yaml | 9 + provisioning/datasources/datasource.yaml | 21 +++ worker/cmd/generate.go | 6 +- 5 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 dashboards/instruct-lab-bot.json create mode 100644 docker-compose.yml create mode 100644 provisioning/dashboards/dashboard.yaml create mode 100644 provisioning/datasources/datasource.yaml diff --git a/dashboards/instruct-lab-bot.json b/dashboards/instruct-lab-bot.json new file mode 100644 index 00000000..d001ac30 --- /dev/null +++ b/dashboards/instruct-lab-bot.json @@ -0,0 +1,231 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "redis-datasource", + "uid": "PA7F6415749A3297A" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "command": "llen", + "datasource": { + "type": "redis-datasource", + "uid": "PA7F6415749A3297A" + }, + "keyName": "generate", + "query": "", + "refId": "A", + "streaming": true, + "type": "command" + } + ], + "title": "Generate Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "redis-datasource", + "uid": "PA7F6415749A3297A" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "command": "llen", + "datasource": { + "type": "redis-datasource", + "uid": "PA7F6415749A3297A" + }, + "keyName": "results", + "query": "", + "refId": "A", + "streaming": true, + "type": "command" + } + ], + "title": "Result Queue Length", + "type": "timeseries" + } + ], + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Instruct Lab Bot", + "uid": "EDm3MDJIz", + "version": 1, + "weekStart": "" +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..56ebe42d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3.4" + +services: + redis: + container_name: redis + image: redis:latest + ports: + - 6379:6379 + + grafana: + container_name: grafana + image: ghcr.io/redisgrafana/redis-app:latest + ports: + - 3333:3000 + environment: + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_BASIC_ENABLED=false + - GF_ENABLE_GZIP=true + - GF_USERS_DEFAULT_THEME=light + volumes: + - ./provisioning:/etc/grafana/provisioning + - ./dashboards:/etc/dashboards diff --git a/provisioning/dashboards/dashboard.yaml b/provisioning/dashboards/dashboard.yaml new file mode 100644 index 00000000..9fc8aa71 --- /dev/null +++ b/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + updateIntervalSeconds: 30 + options: + path: /etc/dashboards + foldersFromFilesStructure: true diff --git a/provisioning/datasources/datasource.yaml b/provisioning/datasources/datasource.yaml new file mode 100644 index 00000000..ded98477 --- /dev/null +++ b/provisioning/datasources/datasource.yaml @@ -0,0 +1,21 @@ +apiVersion: 1 + +deleteDatasources: + - name: Redis + orgId: 1 + +datasources: + - name: Redis + type: redis-datasource + access: proxy + orgId: 1 + isDefault: true + version: 1 + url: redis://redis:6379 + jsonData: + client: standalone + poolSize: 5 + timeout: 10 + pingInterval: 0 + pipelineWindow: 0 + editable: true diff --git a/worker/cmd/generate.go b/worker/cmd/generate.go index a6d42498..2808c0bb 100644 --- a/worker/cmd/generate.go +++ b/worker/cmd/generate.go @@ -112,11 +112,7 @@ var generateCmd = &cobra.Command{ go func(ch <-chan string) { defer wg.Done() for job := range jobChan { - wg.Add(1) - go func(job string) { - defer wg.Done() - processJob(ctx, conn, svc, sugar, job) - }(job) + processJob(ctx, conn, svc, sugar, job) } }(jobChan)