Skip to content

Commit

Permalink
Add grafana dashboard for monitoring queues
Browse files Browse the repository at this point in the history 
Fixes a bug in the worker where a single worker would take all
available work!!!

Adds a docker-compose.yml for using Grafana + a dashboard that
streams the queue length from Redis.

Signed-off-by: Dave Tucker <[email protected]>
  • Loading branch information
@dave-tucker
dave-tucker committed Mar 17, 2024
1 parent 0cad5d9 commit 1ccc436
Show file tree
Hide file tree
Showing 5 changed files with 285 additions and 5 deletions.
231 changes: 231 additions & 0 deletions dashboards/instruct-lab-bot.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,231 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "redis-datasource",
"uid": "PA7F6415749A3297A"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"command": "llen",
"datasource": {
"type": "redis-datasource",
"uid": "PA7F6415749A3297A"
},
"keyName": "generate",
"query": "",
"refId": "A",
"streaming": true,
"type": "command"
}
],
"title": "Generate Queue Length",
"type": "timeseries"
},
{
"datasource": {
"type": "redis-datasource",
"uid": "PA7F6415749A3297A"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"command": "llen",
"datasource": {
"type": "redis-datasource",
"uid": "PA7F6415749A3297A"
},
"keyName": "results",
"query": "",
"refId": "A",
"streaming": true,
"type": "command"
}
],
"title": "Result Queue Length",
"type": "timeseries"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Instruct Lab Bot",
"uid": "EDm3MDJIz",
"version": 1,
"weekStart": ""
}
23 changes: 23 additions & 0 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
version: "3.4"

services:
redis:
container_name: redis
image: redis:latest
ports:
- 6379:6379

grafana:
container_name: grafana
image: ghcr.io/redisgrafana/redis-app:latest
ports:
- 3333:3000
environment:
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_BASIC_ENABLED=false
- GF_ENABLE_GZIP=true
- GF_USERS_DEFAULT_THEME=light
volumes:
- ./provisioning:/etc/grafana/provisioning
- ./dashboards:/etc/dashboards
9 changes: 9 additions & 0 deletions provisioning/dashboards/dashboard.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: 1

providers:
- name: dashboards
type: file
updateIntervalSeconds: 30
options:
path: /etc/dashboards
foldersFromFilesStructure: true
21 changes: 21 additions & 0 deletions provisioning/datasources/datasource.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
apiVersion: 1

deleteDatasources:
- name: Redis
orgId: 1

datasources:
- name: Redis
type: redis-datasource
access: proxy
orgId: 1
isDefault: true
version: 1
url: redis://redis:6379
jsonData:
client: standalone
poolSize: 5
timeout: 10
pingInterval: 0
pipelineWindow: 0
editable: true
6 changes: 1 addition & 5 deletions worker/cmd/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,7 @@ var generateCmd = &cobra.Command{
go func(ch <-chan string) {
defer wg.Done()
for job := range jobChan {
wg.Add(1)
go func(job string) {
defer wg.Done()
processJob(ctx, conn, svc, sugar, job)
}(job)
processJob(ctx, conn, svc, sugar, job)
}
}(jobChan)

Expand Down

0 comments on commit 1ccc436

Please sign in to comment.