diff --git a/.idea/deep-helm.iml b/.idea/deep-helm.iml index 2cdb1e3..a4de2ac 100644 --- a/.idea/deep-helm.iml +++ b/.idea/deep-helm.iml @@ -1,5 +1,6 @@ + diff --git a/charts/deep-distributed/Chart.yaml b/charts/deep-distributed/Chart.yaml index 9b41965..0e204f0 100644 --- a/charts/deep-distributed/Chart.yaml +++ b/charts/deep-distributed/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 +version: 1.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/deep-distributed/values.yaml b/charts/deep-distributed/values.yaml index 89ef834..45ce1bb 100644 --- a/charts/deep-distributed/values.yaml +++ b/charts/deep-distributed/values.yaml @@ -1032,7 +1032,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/unhealthy_ring_node/ summary: One or more ring members are unhealthy - alert: Required Ring Member Missing - expr: deep_ring_members{state="ACTIVE", name=~"ingester|tracepoint"} == 0 + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name=~"ingester|tracepoint"} == 0 labels: severity: critical annotations: @@ -1040,7 +1040,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Tracepoint ring missing member - expr: deep_ring_members{state="ACTIVE", name="tracepoint"} != {{ .Values.tracepoint.replicas }} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="tracepoint"} != {{ .Values.tracepoint.replicas }} labels: severity: critical annotations: @@ -1048,7 +1048,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Ingester ring missing member - expr: deep_ring_members{state="ACTIVE", name="ingester"} != {{ .Values.ingester.replicas }} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="ingester"} != {{ .Values.ingester.replicas }} labels: severity: critical annotations: @@ -1056,7 +1056,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Compactor ring missing member - expr: deep_ring_members{state="ACTIVE", name="compactor"} != {{ .Values.compactor.replicas }} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="compactor"} != {{ .Values.compactor.replicas }} labels: severity: critical annotations: @@ -1064,13 +1064,22 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Metrics Generator ring missing member - expr: deep_ring_members{state="ACTIVE", name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.distributor.replicas }} {{else}}0{{end}} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.distributor.replicas }} {{else}}0{{end}} labels: severity: critical annotations: description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}." runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes + - alert: Rapid increase in blocks + expr: delta(deep_db_blocklist_length{cluster="{{ include "deep.clusterName" (dict "ctx" $)}}"}[24h]) > 30 + for: 48h + labels: + severity: warning + annotations: + description: "Tenant: {{ $labels.tenant }} has seen a large increase in blocks: {{ $value }}." + runbook_url: https://intergral.github.io/deep/runbooks/block_increase/ + summary: Tenant has seen large increase in blocks. # Configuration for the gateway diff --git a/charts/deep/Chart.yaml b/charts/deep/Chart.yaml index ede85a1..bef7cd5 100644 --- a/charts/deep/Chart.yaml +++ b/charts/deep/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.14 +version: 1.0.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/deep/dashboards/deep-operational.json b/charts/deep/dashboards/deep-operational.json index 8105f6f..f3f455e 100644 --- a/charts/deep/dashboards/deep-operational.json +++ b/charts/deep/dashboards/deep-operational.json @@ -139,7 +139,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])", + "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])", "interval": "", "legendFormat": "{{ "{{pod}}" }}", "range": true, @@ -227,7 +227,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}", "interval": "", "legendFormat": "{{ "{{pod}}" }}", "refId": "A" @@ -314,7 +314,7 @@ "pluginVersion": "9.0.0-d373beebpre", "targets": [ { - "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}", "legendFormat": "{{ "{{pod}}" }}", "refId": "A" } @@ -904,7 +904,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "deep_db_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"} / deep_db_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "deep_db_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"} / deep_db_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}", "legendFormat": "{{ "{{instance}}" }}", "range": true, "refId": "A" @@ -997,7 +997,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(deep_db_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "expr": "sum(increase(deep_db_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (job)", "legendFormat": "compaction_err", "range": true, "refId": "B" @@ -1008,7 +1008,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(deep_db_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "expr": "sum(increase(deep_db_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (job)", "legendFormat": "retention_err", "range": true, "refId": "C" @@ -1019,7 +1019,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(deep_db_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "expr": "sum(increase(deep_db_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (job)", "legendFormat": "blocklist_err", "range": true, "refId": "D" @@ -1112,7 +1112,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_db_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_db_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -1124,7 +1124,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_db_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_db_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (le))", "legendFormat": ".9", "range": true, "refId": "B" @@ -1135,7 +1135,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_db_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_db_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -1230,7 +1230,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "avg(deep_db_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}) by (tenant)", + "expr": "avg(deep_db_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}) by (tenant)", "instant": false, "interval": "", "legendFormat": "{{ "{{tenant}}" }}", @@ -1324,7 +1324,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_db_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_db_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -1336,7 +1336,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_db_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_db_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -1348,7 +1348,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_db_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_db_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -1442,7 +1442,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(deep_db_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "expr": "sum(increase(deep_db_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval]))", "interval": "", "legendFormat": "deleted", "range": true, @@ -1454,7 +1454,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(deep_db_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "expr": "sum(increase(deep_db_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval]))", "interval": "", "legendFormat": "marked_for_deletion", "range": true, @@ -1665,7 +1665,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{route=~\".*api_snapshots_snapshotid\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{route=~\".*api_snapshots_snapshotid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -1761,7 +1761,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -1773,7 +1773,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -1785,7 +1785,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -1881,7 +1881,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(deep_request_duration_seconds_bucket{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_bucket{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -1979,7 +1979,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -1993,7 +1993,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "hide": false, "interval": "", "legendFormat": ".9", @@ -2008,7 +2008,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "hide": false, "interval": "", "legendFormat": ".5", @@ -2105,7 +2105,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_snapshots_snapshotid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_snapshots_snapshotid\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -2201,7 +2201,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -2213,7 +2213,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -2225,7 +2225,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -2321,7 +2321,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -2419,7 +2419,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -2433,7 +2433,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -2447,7 +2447,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -2543,7 +2543,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_bucket{route=~\"querier_.*api_snapshots_snapshotid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_bucket{route=~\"querier_.*api_snapshots_snapshotid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -2639,7 +2639,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -2651,7 +2651,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -2663,7 +2663,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_snapshots_snapshotid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -2759,7 +2759,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(deep_request_duration_seconds_bucket{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_bucket{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -2857,7 +2857,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -2871,7 +2871,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -2885,7 +2885,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -2981,7 +2981,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_bucket{route=\"/deeppb.QuerierService/FindSnapshotByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_bucket{route=\"/deeppb.QuerierService/FindSnapshotByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -3077,7 +3077,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/deeppb.QuerierService/FindSnapshotByID\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"/deeppb.QuerierService/FindSnapshotByID\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3089,7 +3089,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/deeppb.QuerierService/FindSnapshotByID\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"/deeppb.QuerierService/FindSnapshotByID\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3101,7 +3101,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/deeppb.QuerierService/FindSnapshotByID\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"/deeppb.QuerierService/FindSnapshotByID\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3197,7 +3197,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(deep_request_duration_seconds_count{route=~\"/deeppb.QuerierService/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{route=~\"/deeppb.QuerierService/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -3295,7 +3295,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/deeppb.QuerierService/Search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.QuerierService/Search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -3309,7 +3309,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/deeppb.QuerierService/Search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.QuerierService/Search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -3323,7 +3323,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/deeppb.QuerierService/Search.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.QuerierService/Search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -3418,7 +3418,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{route=~\".*api_metrics_summary\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{route=~\".*api_metrics_summary\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -3514,7 +3514,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "queryType": "search", @@ -3527,7 +3527,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "queryType": "search", @@ -3540,7 +3540,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/.*\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_metrics_summary\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "queryType": "search", @@ -3636,7 +3636,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_metrics.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_metrics.*\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -3734,7 +3734,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "queryType": "search", @@ -3749,7 +3749,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "queryType": "search", @@ -3764,7 +3764,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\".*api_metrics.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "queryType": "search", @@ -3860,7 +3860,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{route=~\"querier_.*api_metrics.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{route=~\"querier_.*api_metrics.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }} {{ "{{route}}" }}", @@ -3956,7 +3956,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "queryType": "search", @@ -3969,7 +3969,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "queryType": "search", @@ -3982,7 +3982,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"querier_.*api_metrics.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "queryType": "search", @@ -4077,7 +4077,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{route=\"/deeppb.MetricsGenerator/GetMetrics\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{route=\"/deeppb.MetricsGenerator/GetMetrics\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{ "{{status_code}}" }}", @@ -4173,7 +4173,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/deeppb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"/deeppb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "queryType": "search", @@ -4186,7 +4186,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/deeppb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"/deeppb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "queryType": "search", @@ -4199,7 +4199,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=\"/deeppb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"/deeppb.MetricsGenerator/GetMetrics\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "queryType": "search", @@ -4498,7 +4498,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(deep_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1h]))", + "expr": "sum(increase(deep_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[1h]))", "interval": "", "legendFormat": "{{ "{{pod}}" }}", "range": true, @@ -4898,7 +4898,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)", "interval": "", "legendFormat": "{{ "{{status_code}}" }}", "range": true, @@ -5098,7 +5098,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -5110,7 +5110,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -5122,7 +5122,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -5216,7 +5216,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "range": true, @@ -5228,7 +5228,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "range": true, @@ -5240,7 +5240,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "range": true, @@ -5442,7 +5442,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/deeppb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "queryType": "search", @@ -5455,7 +5455,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/deeppb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "queryType": "search", @@ -5468,7 +5468,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/deeppb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(deep_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\", route=~\"/deeppb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "queryType": "search", @@ -5582,7 +5582,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)", + "expr": "sum(rate(deep_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code, method)", "interval": "", "legendFormat": "{{ "{{status_code}}" }}-{{ "{{reason}}" }}", "range": true, @@ -5677,7 +5677,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".99-{{ "{{reason}}" }}", "range": true, @@ -5689,7 +5689,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.9, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".9-{{ "{{reason}}" }}", "range": true, @@ -5701,7 +5701,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "expr": "histogram_quantile(.5, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (method, le))", "interval": "", "legendFormat": ".5-{{ "{{reason}}" }}", "range": true, @@ -5814,7 +5814,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_db_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(deep_db_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (status_code, operation)", "interval": "", "legendFormat": "{{ "{{status_code}}" }}-{{ "{{operation}}" }}", "range": true, @@ -5909,7 +5909,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (operation, le))", "legendFormat": ".99-{{ "{{operation}}" }}", "range": true, "refId": "A" @@ -5920,7 +5920,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.9, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (operation, le))", "legendFormat": ".9-{{ "{{operation}}" }}", "range": true, "refId": "B" @@ -5931,7 +5931,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "histogram_quantile(.5, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (operation, le))", "legendFormat": ".5-{{ "{{operation}}" }}", "range": true, "refId": "C" @@ -6043,7 +6043,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}", "interval": "", "legendFormat": "{{ "{{instance}}" }}", "queryType": "search", @@ -6139,7 +6139,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "deep_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "expr": "deep_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}", "interval": "", "legendFormat": "{{ "{{instance}}" }}", "range": true, @@ -6234,7 +6234,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "min(deep_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "min(deep_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"})", "interval": "", "legendFormat": "min", "range": true, @@ -6246,7 +6246,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "max(deep_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "max(deep_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"})", "interval": "", "legendFormat": "max", "range": true, @@ -6341,7 +6341,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "min(deep_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "min(deep_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"})", "interval": "", "legendFormat": "min", "queryType": "search", @@ -6354,7 +6354,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "max(deep_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "max(deep_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"})", "interval": "", "legendFormat": "max", "queryType": "search", @@ -6641,7 +6641,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_db_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(deep_db_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "range": true, @@ -6736,7 +6736,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(deep_db_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(deep_db_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "range": true, @@ -6826,7 +6826,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "sum(rate(deep_db_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(deep_db_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "__auto", "queryType": "search", @@ -6917,7 +6917,7 @@ "uid": "${ds}" }, "editorMode": "code", - "expr": "sum(increase(deep_db_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[5m])) by (level)", + "expr": "sum(increase(deep_db_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"deep\"}[5m])) by (level)", "interval": "", "legendFormat": "__auto", "queryType": "search", diff --git a/charts/deep/dashboards/deep-reads.json b/charts/deep/dashboards/deep-reads.json index aa792cc..056e9ec 100644 --- a/charts/deep/dashboards/deep-reads.json +++ b/charts/deep/dashboards/deep-reads.json @@ -255,7 +255,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -354,7 +354,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -368,7 +368,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -381,7 +381,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__interval])) by (route)", + "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -611,7 +611,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -710,7 +710,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -724,7 +724,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -738,7 +738,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__interval])) by (route)", + "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",route=~\"api_.*\"}[$__interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -969,7 +969,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1068,7 +1068,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1082,7 +1082,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"querier_api_.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1096,7 +1096,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__interval])) by (route)", + "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",route=~\"querier_api_.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",route=~\"querier_api_.*\"}[$__interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1327,7 +1327,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1426,7 +1426,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (le,endpoint)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by (le,endpoint)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1440,7 +1440,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (le,endpoint)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_querier_external_endpoint_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by (le,endpoint)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1454,7 +1454,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (endpoint) * 1e3 / sum(rate(deep_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__interval])) by (endpoint)", + "expr": "sum(rate(deep_querier_external_endpoint_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by (endpoint) * 1e3 / sum(rate(deep_querier_external_endpoint_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by (endpoint)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1685,7 +1685,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/deeppb.QuerierService/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"/deeppb.QuerierService/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1784,7 +1784,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1798,7 +1798,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (le,route)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (le,route)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1812,7 +1812,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (route)", + "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (route) * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.QuerierService/.*\"}[$__interval])) by (route)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2043,7 +2043,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2142,7 +2142,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2156,7 +2156,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2170,7 +2170,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by () * 1e3 / sum(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by ()", + "expr": "sum(rate(deep_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by () * 1e3 / sum(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2401,7 +2401,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2500,7 +2500,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2514,7 +2514,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",operation=\"GET\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2528,7 +2528,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_db_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by () * 1e3 / sum(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by ()", + "expr": "sum(rate(deep_db_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",operation=\"GET\"}[$__interval])) by () * 1e3 / sum(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",operation=\"GET\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/charts/deep/dashboards/deep-resource.json b/charts/deep/dashboards/deep-resource.json index ea40920..1421e86 100644 --- a/charts/deep/dashboards/deep-resource.json +++ b/charts/deep/dashboards/deep-resource.json @@ -483,7 +483,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod, container) (go_memstats_heap_inuse_bytes{ job=~\"($namespace)/.*\"})", + "expr": "sum by(pod, container) (go_memstats_heap_inuse_bytes{ job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -940,7 +940,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{job=~\"($namespace)/distributor\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1383,7 +1383,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=~\"($namespace)/ingester\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1826,7 +1826,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=~\"($namespace)/metrics-generator\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2269,7 +2269,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=~\"($namespace)/query-frontend\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2712,7 +2712,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=~\"($namespace)/querier\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -3155,7 +3155,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=~\"($namespace)/compactor\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{ job=\"deep\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/charts/deep/dashboards/deep-rollout.json b/charts/deep/dashboards/deep-rollout.json index ac2c28f..5dacfe5 100644 --- a/charts/deep/dashboards/deep-rollout.json +++ b/charts/deep/dashboards/deep-rollout.json @@ -253,7 +253,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", "instant": false, "interval": "", "legendFormat": "", @@ -378,7 +378,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", "instant": false, "interval": "", "legendFormat": "", @@ -499,7 +499,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]))\n", "instant": false, "interval": "", "legendFormat": "", @@ -622,7 +622,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))\n", + "expr": "histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))\n", "instant": false, "interval": "", "legendFormat": "", @@ -739,7 +739,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\"}[$__rate_interval]))\n", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\"}[$__rate_interval]))\n", "instant": false, "interval": "", "legendFormat": "", @@ -864,7 +864,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\"}[$__rate_interval]))\n", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\"}[$__rate_interval]))\n", "instant": false, "interval": "", "legendFormat": "", @@ -985,7 +985,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\"}[$__rate_interval]))\n", + "expr": "sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\"}[$__rate_interval]))\n", "instant": false, "interval": "", "legendFormat": "", @@ -1108,7 +1108,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"deep_api_.*\"}))", + "expr": "histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\"}))", "instant": false, "interval": "", "legendFormat": "", @@ -1475,7 +1475,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/.*\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))[1h:])\n)\n", + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=~\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}))[1h:])\n)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1489,7 +1489,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"deep_api_.*\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"deep_api_.*\"}))[1h:])\n)\n", + "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=~\"deep_api_.*\"}))[1h:])\n)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/charts/deep/dashboards/deep-tenants.json b/charts/deep/dashboards/deep-tenants.json index f8569e4..40b10f4 100644 --- a/charts/deep/dashboards/deep-tenants.json +++ b/charts/deep/dashboards/deep-tenants.json @@ -143,7 +143,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n) by (limit_name)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=\"deep\",user=\"$tenant\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=\"deep\"})\n) by (limit_name)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -320,7 +320,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_distributor_snapshot_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))", + "expr": "sum(rate(deep_distributor_snapshot_bytes_received_total{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -334,7 +334,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_rate_limit_bytes\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_rate_limit_bytes\"})\n) by (ingestion_rate_limit_bytes)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=\"deep\",user=\"$tenant\",limit_name=\"ingestion_rate_limit_bytes\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=\"deep\",limit_name=\"ingestion_rate_limit_bytes\"})\n) by (ingestion_rate_limit_bytes)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -348,7 +348,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"ingestion_burst_size_bytes\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"ingestion_burst_size_bytes\"})\n) by (ingestion_burst_size_bytes)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=\"deep\",user=\"$tenant\",limit_name=\"ingestion_burst_size_bytes\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=\"deep\",limit_name=\"ingestion_burst_size_bytes\"})\n) by (ingestion_burst_size_bytes)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -447,7 +447,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_distributor_snapshots_requests{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval]))", + "expr": "sum(rate(deep_distributor_snapshots_requests{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -461,7 +461,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_discarded_snapshots_total{cluster=~\"$cluster\", job=~\"($namespace)/distributor\",tenant=\"$tenant\"}[$__rate_interval])) by (reason)", + "expr": "sum(rate(deep_discarded_snapshots_total{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"}[$__rate_interval])) by (reason)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -605,7 +605,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(deep_ingester_live_snapshots{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",tenant=\"$tenant\"})", + "expr": "max(deep_ingester_live_snapshots{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -619,7 +619,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_global_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_global_traces_per_user\"})\n) by (max_global_traces_per_user)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=\"deep\",user=\"$tenant\",limit_name=\"max_global_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=\"deep\",limit_name=\"max_global_traces_per_user\"})\n) by (max_global_traces_per_user)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -633,7 +633,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"max_local_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"max_local_traces_per_user\"})\n) by (max_local_traces_per_user)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=\"deep\",user=\"$tenant\",limit_name=\"max_local_traces_per_user\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=\"deep\",limit_name=\"max_local_traces_per_user\"})\n) by (max_local_traces_per_user)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -758,7 +758,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"snapshots\"}[$__rate_interval])) by (status)", + "expr": "sum(rate(deep_query_frontend_queries_total{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\",op=\"snapshots\"}[$__rate_interval])) by (status)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -857,7 +857,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",tenant=\"$tenant\",op=\"search\"}[$__rate_interval])) by (status)", + "expr": "sum(rate(deep_query_frontend_queries_total{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\",op=\"search\"}[$__rate_interval])) by (status)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -982,7 +982,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "avg(deep_db_blocklist_length{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})", + "expr": "avg(deep_db_blocklist_length{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1081,7 +1081,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(deep_db_compaction_outstanding_blocks{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",tenant=\"$tenant\"})\n/\ncount(deep_build_info{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})\n", + "expr": "sum(deep_db_compaction_outstanding_blocks{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"})\n/\ncount(deep_build_info{cluster=~\"$cluster\", job=\"deep\"})\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1206,7 +1206,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_metrics_generator_bytes_received_total{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"}[$__rate_interval]))", + "expr": "sum(rate(deep_metrics_generator_bytes_received_total{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1328,7 +1328,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(deep_metrics_generator_registry_active_series{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\",tenant=\"$tenant\"})", + "expr": "sum(deep_metrics_generator_registry_active_series{cluster=~\"$cluster\", job=\"deep\",tenant=\"$tenant\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1342,7 +1342,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",user=\"$tenant\",limit_name=\"metrics_generator_max_active_series\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",limit_name=\"metrics_generator_max_active_series\"})\n) by (metrics_generator_max_active_series)\n", + "expr": "max(\n max by (cluster, namespace, limit_name) (deep_limits_overrides{cluster=~\"$cluster\", job=\"deep\",user=\"$tenant\",limit_name=\"metrics_generator_max_active_series\"})\n or max by (cluster, namespace, limit_name) (deep_limits_defaults{cluster=~\"$cluster\", job=\"deep\",limit_name=\"metrics_generator_max_active_series\"})\n) by (metrics_generator_max_active_series)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1462,7 +1462,7 @@ "type": "intergral-deep-datasource", "uid": "$datasource" }, - "definition": "label_values(deep_db_blocklist_length{cluster=\"$cluster\", job=~\"($namespace)/compactor\"},tenant)", + "definition": "label_values(deep_db_blocklist_length{cluster=\"$cluster\", job=\"deep\"},tenant)", "hide": 0, "includeAll": false, "label": "tenant", @@ -1471,7 +1471,7 @@ "options": [], "query": { "labelFilters": [], - "query": "label_values(deep_db_blocklist_length{cluster=\"$cluster\", job=~\"($namespace)/compactor\"},tenant)", + "query": "label_values(deep_db_blocklist_length{cluster=\"$cluster\", job=\"deep\"},tenant)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/charts/deep/dashboards/deep-writes.json b/charts/deep/dashboards/deep-writes.json index 6aa809c..d80b59a 100644 --- a/charts/deep/dashboards/deep-writes.json +++ b/charts/deep/dashboards/deep-writes.json @@ -255,7 +255,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -354,7 +354,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -368,7 +368,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -382,7 +382,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by () * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by ()", + "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by () * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -505,7 +505,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", __name__=~\"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+\"},\n \"grpc_status\", \"$1\", \"__name__\", \"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)\"\n )\n [$__interval:$__interval]\n )\n)\n", + "expr": "sum by (grpc_status) (\n rate(\n label_replace(\n {cluster=~\"$cluster\", job=\"deep\", __name__=~\"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_[0-9]+\"},\n \"grpc_status\", \"$1\", \"__name__\", \"envoy_cluster_grpc_proto_collector_trace_v1_TraceService_(.+)\"\n )\n [$__interval:$__interval]\n )\n)\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -663,7 +663,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_receiver_accepted_snapshots{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval]))", + "expr": "sum(rate(deep_receiver_accepted_snapshots{cluster=~\"$cluster\", job=\"deep\"}[$__interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -677,7 +677,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_receiver_refused_snapshots{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval]))", + "expr": "sum(rate(deep_receiver_refused_snapshots{cluster=~\"$cluster\", job=\"deep\"}[$__interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -776,7 +776,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -790,7 +790,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_distributor_push_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -804,7 +804,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_distributor_push_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by () * 1e3 / sum(rate(deep_distributor_push_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}[$__interval])) by ()", + "expr": "sum(rate(deep_distributor_push_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by () * 1e3 / sum(rate(deep_distributor_push_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1035,7 +1035,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\", route=~\"/deeppb.IngesterService/Push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1096,7 +1096,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1108,7 +1108,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1120,7 +1120,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by () * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by ()", + "expr": "sum(rate(deep_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by () * 1e3 / sum(rate(deep_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",route=~\"/deeppb.IngesterService/Push.*\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1378,7 +1378,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1477,7 +1477,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1491,7 +1491,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1505,7 +1505,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",method=\"Memcache.Put\"}[$__interval])) by ()", + "expr": "sum(rate(deep_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1736,7 +1736,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1835,7 +1835,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1849,7 +1849,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1863,7 +1863,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_db_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",operation=~\"(PUT|POST)\"}[$__interval])) by ()", + "expr": "sum(rate(deep_db_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2094,7 +2094,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2193,7 +2193,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2207,7 +2207,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2221,7 +2221,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",method=\"Memcache.Put\"}[$__interval])) by ()", + "expr": "sum(rate(deep_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by () * 1e3 / sum(rate(deep_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",method=\"Memcache.Put\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2452,7 +2452,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2551,7 +2551,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2565,7 +2565,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(deep_db_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by (le,)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2579,7 +2579,7 @@ "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(deep_db_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor\",operation=~\"(PUT|POST)\"}[$__interval])) by ()", + "expr": "sum(rate(deep_db_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by () * 1e3 / sum(rate(deep_db_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=\"deep\",operation=~\"(PUT|POST)\"}[$__interval])) by ()", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/charts/deep/templates/NOTES.txt b/charts/deep/templates/NOTES.txt index 260fdc7..f792353 100644 --- a/charts/deep/templates/NOTES.txt +++ b/charts/deep/templates/NOTES.txt @@ -7,5 +7,5 @@ Deep version: {{ .Chart.AppVersion }} Installed components: * deep ({{.Values.deep.replicas}}) {{- if .Values.service.enabled}} - * service {{- if .Values.service.ingress.enabled}}{{- with (index .Values.service.ingress.hosts 0)}} {{.host}}{{- end}}{{- end}} + * service {{ if .Values.service.ingress.enabled}}{{- with (index .Values.service.ingress.hosts 0)}} {{.host}}{{- end}}{{- end}} {{ if .Values.service.traefik.enabled}}{{- .Values.service.traefik.host}} {{- end}} {{- end}} \ No newline at end of file diff --git a/charts/deep/values.yaml b/charts/deep/values.yaml index 0e9e2b2..58755c8 100644 --- a/charts/deep/values.yaml +++ b/charts/deep/values.yaml @@ -477,7 +477,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/unhealthy_ring_node/ summary: One or more ring members are unhealthy - alert: Required Ring Member Missing - expr: deep_ring_members{state="ACTIVE", name=~"ingester|tracepoint"} == 0 + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name=~"ingester|tracepoint"} == 0 labels: severity: critical annotations: @@ -485,7 +485,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Tracepoint ring missing member - expr: deep_ring_members{state="ACTIVE", name="tracepoint"} != {{ .Values.deep.replicas }} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="tracepoint"} != {{ .Values.tracepoint.replicas }} labels: severity: critical annotations: @@ -493,7 +493,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Ingester ring missing member - expr: deep_ring_members{state="ACTIVE", name="ingester"} != {{ .Values.deep.replicas }} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="ingester"} != {{ .Values.ingester.replicas }} labels: severity: critical annotations: @@ -501,7 +501,7 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Compactor ring missing member - expr: deep_ring_members{state="ACTIVE", name="compactor"} != {{ .Values.deep.replicas }} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="compactor"} != {{ .Values.compactor.replicas }} labels: severity: critical annotations: @@ -509,10 +509,19 @@ monitoring: runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes - alert: Metrics Generator ring missing member - expr: deep_ring_members{state="ACTIVE", name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.deep.replicas }} {{else}}0{{end}} + expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" (dict "ctx" $)}}", name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.distributor.replicas }} {{else}}0{{end}} labels: severity: critical annotations: description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}." runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/ summary: One or more ring has missing required nodes + - alert: Rapid increase in blocks + expr: delta(deep_db_blocklist_length{cluster="{{ include "deep.clusterName" (dict "ctx" $)}}"}[24h]) > 30 + for: 48h + labels: + severity: warning + annotations: + description: "Tenant: {{ $labels.tenant }} has seen a large increase in blocks: {{ $value }}." + runbook_url: https://intergral.github.io/deep/runbooks/block_increase/ + summary: Tenant has seen large increase in blocks.