feat(container)!: Update kube-prometheus-stack Docker tag to v65

[mchesterbot]

This PR contains the following updates:

Package	Update	Change
kube-prometheus-stack (source)	major	64.0.0 -> 65.0.0

---

Warning

Some dependencies could not be looked up. Check the Dependency Dashboard for more information.

---

Release Notes

prometheus-community/helm-charts (kube-prometheus-stack)

v65.0.0

Compare Source

---

Configuration

📅 Schedule: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined).

🚦 Automerge: Disabled by config. Please merge this manually once you are satisfied.

♻ Rebasing: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox.

🔕 Ignore: Close this PR and you won't be reminded about this update again.

---

• If you want to rebase/retry this PR, check this box

---

This PR has been generated by Renovate Bot.

[mchesterbot]

--- kubernetes/apps/monitoring/kube-prometheus-stack/app Kustomization: flux-system/cluster-apps-kube-prometheus-stack HelmRelease: monitoring/kube-prometheus-stack

+++ kubernetes/apps/monitoring/kube-prometheus-stack/app Kustomization: flux-system/cluster-apps-kube-prometheus-stack HelmRelease: monitoring/kube-prometheus-stack

@@ -13,13 +13,13 @@

       chart: kube-prometheus-stack
       interval: 5m
       sourceRef:
         kind: HelmRepository
         name: prometheus-community-charts
         namespace: flux-system
-      version: 64.0.0
+      version: 65.0.0
   install:
     createNamespace: true
     remediation:
       retries: 3
   interval: 30m
   maxHistory: 2

[mchesterbot]

--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-apiserver

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-apiserver

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  apiserver.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  apiserver.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"description":"The SLO (service level objective) and other metrics
     displayed on this dashboard are for informational purposes only.","gridPos":{"h":2,"w":24,"x":0,"y":0},"id":1,"options":{"content":"The
     SLO (service level objective) and other metrics displayed on this dashboard are
     for informational purposes only."},"pluginVersion":"v11.0.0","title":"Notice","type":"text"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"description":"How many percent of requests (both read and write) in
     30 days have been answered successfully and fast enough?","fieldConfig":{"defaults":{"decimals":3,"unit":"percentunit"}},"gridPos":{"h":7,"w":8,"x":0,"y":2},"id":2,"interval":"1m","pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"apiserver_request:availability30d{verb=\"all\",
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-cluster-total

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-cluster-total

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  cluster-total.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  cluster-total.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"showPoints":"never"},"unit":"binBps"}},"gridPos":{"h":9,"w":12,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum
     by (namespace) (\n    rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n  *
     on (cluster,namespace,pod) group_left ()\n    topk by (cluster,namespace,pod)
     (\n      1,\n      max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n    )\n)\n","legendFormat":"__auto"}],"title":"Current
     Rate of Bytes Received","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"showPoints":"never"},"unit":"binBps"}},"gridPos":{"h":9,"w":12,"x":12,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-cluster

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-cluster

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-cluster.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-cluster.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}","instant":true}],"title":"CPU
     Utilisation","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed
     --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":4,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"})
     / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})","instant":true}],"title":"CPU
     Requests Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":8,"y":0},"id":3,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"})
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-multicluster

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-multicluster

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-multicluster.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-multicluster.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":3,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"cluster:node_cpu:ratio_rate5m","instant":true}],"title":"CPU
     Utilisation","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed
     --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":4,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\",
     resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",
     resource=\"cpu\"})","instant":true}],"title":"CPU Requests Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":8,"y":0},"id":3,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\",
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-namespace

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-namespace

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-namespace.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-namespace.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":6,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
     namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\",
     cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})","instant":true}],"title":"CPU
     Utilisation (from requests)","type":"stat"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":6,"x":6,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
     namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\",
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-node

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-node

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-node.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-node.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true,"stacking":{"mode":"normal"}}},"overrides":[{"matcher":{"id":"byName","options":"max
     capacity"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}},{"id":"custom.stacking","value":{"mode":"none"}},{"id":"custom.hideFrom","value":{"legend":false,"tooltip":true,"viz":false}},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}}]}]},"gridPos":{"h":6,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_node_status_capacity{cluster=\"$cluster\",
     node=~\"$node\", resource=\"cpu\"})","legendFormat":"max capacity"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
     node=~\"$node\"}) by (pod)","legendFormat":"{{pod}}"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Pod"},"properties":[{"id":"links","value":[{"title":"Drill
     down to pods","url":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}"}]}]}]},"gridPos":{"h":6,"w":24,"x":0,"y":6},"id":2,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-pod

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-pod

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-pod.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-pod.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true}},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"custom.lineStyle","value":{"fill":"dash"}},{"id":"custom.lineWidth","value":2},{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]},{"matcher":{"id":"byFrameRefID","options":"C"},"properties":[{"id":"custom.lineStyle","value":{"fill":"dash"}},{"id":"custom.lineWidth","value":2},{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\",
     pod=\"$pod\", cluster=\"$cluster\", container!=\"\"}) by (container)","legendFormat":"__auto"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\",
     cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n","legendFormat":"requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\",
     cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n","legendFormat":"limits"}],"title":"CPU
     Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
     --"},"fieldConfig":{"defaults":{"custom":{"axisColorMode":"thresholds","axisSoftMax":1,"axisSoftMin":0,"fillOpacity":10,"showPoints":"never","spanNulls":true,"thresholdsStyle":{"mode":"dashed+area"}},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":0.25}]}},{"id":"color","value":{"mode":"thresholds","seriesBy":"lastNotNull"}}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(increase(container_cpu_cfs_throttled_periods_total{job=\"kubelet\",
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-workload

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-workload

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-workload.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-workload.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true}}},"gridPos":{"h":7,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n    node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
     namespace=\"$namespace\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type)
     namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\",
     workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","legendFormat":"__auto"}],"title":"CPU
     Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
     --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Pod"},"properties":[{"id":"links","value":[{"title":"Drill
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-workloads-namespace

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-k8s-resources-workloads-namespace

@@ -10,21 +10,21 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  k8s-resources-workloads-namespace.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  k8s-resources-workloads-namespace.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true}},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"custom.lineStyle","value":{"fill":"dash"}},{"id":"custom.lineWidth","value":2},{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]},{"matcher":{"id":"byFrameRefID","options":"C"},"properties":[{"id":"custom.lineStyle","value":{"fill":"dash"}},{"id":"custom.lineWidth","value":2},{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
     namespace=\"$namespace\"}\n* on(namespace,pod)\n  group_left(workload, workload_type)
     namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\",
     workload_type=~\"$type\"}\n) by (workload, workload_type)\n","legendFormat":"{{workload}}
     - {{workload_type}}"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"scalar(kube_resourcequota{cluster=\"$cluster\",
-    namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})","legendFormat":"quota
+    namespace=\"$namespace\", type=\"hard\",resource=~\"requests.cpu|cpu\"})","legendFormat":"quota
     - requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"scalar(kube_resourcequota{cluster=\"$cluster\",
-    namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})","legendFormat":"quota
+    namespace=\"$namespace\", type=\"hard\",resource=~\"limits.cpu\"})","legendFormat":"quota
     - limits"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Workload"},"properties":[{"id":"links","value":[{"title":"Drill
     down to workloads","url":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-type=${__data.fields.Type}&var-workload=${__data.fields.Workload}"}]}]},{"matcher":{"id":"byName","options":"Running
     Pods"},"properties":[{"id":"unit","value":"none"}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":2,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",
     namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload, workload_type)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",
     namespace=\"$namespace\"}\n* on(namespace,pod)\n  group_left(workload, workload_type)
@@ -61,15 +61,15 @@

     --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"custom.lineStyle","value":{"fill":"dash"}},{"id":"custom.lineWidth","value":2},{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]},{"matcher":{"id":"byFrameRefID","options":"C"},"properties":[{"id":"custom.lineStyle","value":{"fill":"dash"}},{"id":"custom.lineWidth","value":2},{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n    container_memory_working_set_bytes{job=\"kubelet\",
     metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",
     container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload,
     workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",
     namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","legendFormat":"{{workload}}
     - {{workload_type}}"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"scalar(kube_resourcequota{cluster=\"$cluster\",
-    namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})","legendFormat":"quota
+    namespace=\"$namespace\", type=\"hard\",resource=~\"requests.memory|memory\"})","legendFormat":"quota
     - requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"scalar(kube_resourcequota{cluster=\"$cluster\",
-    namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})","legendFormat":"quota
+    namespace=\"$namespace\", type=\"hard\",resource=~\"limits.memory\"})","legendFormat":"quota
     - limits"}],"title":"Memory Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Workload"},"properties":[{"id":"links","value":[{"title":"Drill
     down to workloads","url":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-type=${__data.fields.Type}&var-workload=${__data.fields.Workload}"}]}]},{"matcher":{"id":"byName","options":"Running
     Pods"},"properties":[{"id":"unit","value":"none"}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":21},"id":4,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",
     namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload, workload_type)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(\n    container_memory_working_set_bytes{job=\"kubelet\",
     metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-kubelet

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-kubelet

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  kubelet.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  kubelet.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kubelet_node_name{cluster=\"$cluster\",
     job=\"kubelet\", metrics_path=\"/metrics\"})","instant":true}],"title":"Running
     Kubelets","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":4,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kubelet_running_pods{cluster=\"$cluster\",
     job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})","instant":true}],"title":"Running
     Pods","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":8,"y":0},"id":3,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kubelet_running_containers{cluster=\"$cluster\",
     job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})","instant":true}],"title":"Running
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-namespace-by-pod

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-namespace-by-pod

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  namespace-by-pod.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  namespace-by-pod.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"displayName":"$namespace","max":10000000000,"min":0,"thresholds":{"steps":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}]},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":0,"y":0},"id":1,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum
     (\n    rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n  *
     on (cluster,namespace,pod) group_left ()\n    topk by (cluster,namespace,pod)
     (\n      1,\n      max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n    )\n)\n","legendFormat":"__auto"}],"title":"Current
     Rate of Bytes Received","type":"gauge"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"displayName":"$namespace","max":10000000000,"min":0,"thresholds":{"steps":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}]},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":12,"y":0},"id":2,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-namespace-by-workload

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-namespace-by-workload

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  namespace-by-workload.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  namespace-by-workload.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"color":{"fixedColor":"green","mode":"fixed"},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":0,"y":0},"id":1,"options":{"displayMode":"basic","showUnfilled":false},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n*
     on (cluster,namespace,pod) group_left ()\n    topk by (cluster,namespace,pod)
     (\n      1,\n      max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n    )\n*
     on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\",
     workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","legendFormat":"__auto"}],"title":"Current
     Rate of Bytes Received","type":"bargauge"},{"datasource":{"type":"datasource","uid":"--
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-persistentvolumesusage

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-persistentvolumesusage

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  persistentvolumesusage.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  persistentvolumesusage.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":18,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"(\n  sum
     without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\",
     job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n  -\n  sum
     without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\",
     job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n","legendFormat":"Used
     Space"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-pod-total

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-pod-total

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  pod-total.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  pod-total.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"displayName":"$pod","max":10000000000,"min":0,"thresholds":{"steps":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}]},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":0,"y":0},"id":1,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\",
     pod=~\"$pod\"}[$__rate_interval]))","legendFormat":"__auto"}],"title":"Current
     Rate of Bytes Received","type":"gauge"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"displayName":"$pod","max":10000000000,"min":0,"thresholds":{"steps":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}]},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":12,"y":0},"id":2,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\",
     pod=~\"$pod\"}[$__rate_interval]))","legendFormat":"__auto"}],"title":"Current
     Rate of Bytes Transmitted","type":"gauge"},{"datasource":{"type":"datasource","uid":"--
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-workload-total

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-workload-total

@@ -10,13 +10,13 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/instance: kube-prometheus-stack
     app.kubernetes.io/part-of: kube-prometheus-stack
     release: kube-prometheus-stack
     heritage: Helm
 data:
-  workload-total.json: '{"editable":true,"panels":[{"datasource":{"type":"datasource","uid":"--
+  workload-total.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"color":{"fixedColor":"green","mode":"fixed"},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":0,"y":0},"id":1,"options":{"displayMode":"basic","showUnfilled":false},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sort_desc(sum(rate(container_network_receive_bytes_total{job=\"kubelet\",
     metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n*
     on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\",
     workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","legendFormat":"__auto"}],"title":"Current
     Rate of Bytes Received","type":"bargauge"},{"datasource":{"type":"datasource","uid":"--
     Mixed --"},"fieldConfig":{"defaults":{"color":{"fixedColor":"green","mode":"fixed"},"unit":"Bps"}},"gridPos":{"h":9,"w":12,"x":12,"y":0},"id":2,"options":{"displayMode":"basic","showUnfilled":false},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sort_desc(sum(rate(container_network_transmit_bytes_total{job=\"kubelet\",
--- HelmRelease: monitoring/kube-prometheus-stack Deployment: monitoring/kube-prometheus-stack-operator

+++ HelmRelease: monitoring/kube-prometheus-stack Deployment: monitoring/kube-prometheus-stack-operator

@@ -31,18 +31,18 @@

         app: kube-prometheus-stack-operator
         app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator
         app.kubernetes.io/component: prometheus-operator
     spec:
       containers:
       - name: kube-prometheus-stack
-        image: quay.io/prometheus-operator/prometheus-operator:v0.76.1
+        image: quay.io/prometheus-operator/prometheus-operator:v0.77.1
         imagePullPolicy: IfNotPresent
         args:
         - --kubelet-service=kube-system/kube-prometheus-stack-kubelet
         - --localhost=127.0.0.1
-        - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.76.1
+        - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.77.1
         - --config-reloader-cpu-request=11m
         - --config-reloader-cpu-limit=0
         - --config-reloader-memory-request=32M
         - --config-reloader-memory-limit=32M
         - --thanos-default-base-image=quay.io/thanos/thanos:v0.36.1
         - --secret-field-selector=type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1
--- HelmRelease: monitoring/kube-prometheus-stack PrometheusRule: monitoring/kube-prometheus-stack-kube-apiserver-slos

+++ HelmRelease: monitoring/kube-prometheus-stack PrometheusRule: monitoring/kube-prometheus-stack-kube-apiserver-slos

@@ -18,57 +18,57 @@

     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
-        and
-        sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate1h) > (14.40 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate5m) > (14.40 * 0.01000)
       for: 2m
       labels:
         long: 1h
         severity: critical
         short: 5m
     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
-        and
-        sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate6h) > (6.00 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate30m) > (6.00 * 0.01000)
       for: 15m
       labels:
         long: 6h
         severity: critical
         short: 30m
     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
-        and
-        sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate1d) > (3.00 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate2h) > (3.00 * 0.01000)
       for: 1h
       labels:
         long: 1d
         severity: warning
         short: 2h
     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
-        and
-        sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate3d) > (1.00 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate6h) > (1.00 * 0.01000)
       for: 3h
       labels:
         long: 3d
         severity: warning
         short: 6h
 
--- HelmRelease: monitoring/kube-prometheus-stack PrometheusRule: monitoring/kube-prometheus-stack-kubernetes-apps

+++ HelmRelease: monitoring/kube-prometheus-stack PrometheusRule: monitoring/kube-prometheus-stack-kubernetes-apps

@@ -126,13 +126,13 @@

         description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }}
           update has not been rolled out.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
         summary: StatefulSet update has not been rolled out.
       expr: |-
         (
-          max by (namespace, statefulset) (
+          max by (namespace, statefulset, job, cluster) (
             kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
               unless
             kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
           )
             *
           (
--- HelmRelease: monitoring/kube-prometheus-stack PrometheusRule: monitoring/kube-prometheus-stack-kubernetes-system-apiserver

+++ HelmRelease: monitoring/kube-prometheus-stack PrometheusRule: monitoring/kube-prometheus-stack-kubernetes-system-apiserver

@@ -19,25 +19,27 @@

       annotations:
         description: A client certificate used to authenticate to kubernetes apiserver
           is expiring in less than 7.0 days.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
         summary: Client certificate is about to expire.
       expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"}
-        > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
+        > 0 and on (cluster, job) histogram_quantile(0.01, sum by (cluster, job, le)
+        (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
         < 604800
       for: 5m
       labels:
         severity: warning
     - alert: KubeClientCertificateExpiration
       annotations:
         description: A client certificate used to authenticate to kubernetes apiserver
           is expiring in less than 24.0 hours.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
         summary: Client certificate is about to expire.
       expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"}
-        > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
+        > 0 and on (cluster, job) histogram_quantile(0.01, sum by (cluster, job, le)
+        (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
         < 86400
       for: 5m
       labels:
         severity: critical
     - alert: KubeAggregatedAPIErrors
       annotations:
@@ -74,13 +76,14 @@

       annotations:
         description: The kubernetes apiserver has terminated {{ $value | humanizePercentage
           }} of its incoming requests.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests
         summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage
           }} of its incoming requests.
-      expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))  /
-        (  sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
+      expr: sum by (cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
+        / ( sum by (cluster) (rate(apiserver_request_total{job="apiserver"}[10m]))
+        + sum by (cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
         ) > 0.20
       for: 5m
       labels:
         severity: warning
 
--- HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-nodes-aix

+++ HelmRelease: monitoring/kube-prometheus-stack ConfigMap: monitoring/kube-prometheus-stack-nodes-aix

@@ -0,0 +1,75 @@

+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: monitoring
+  name: kube-prometheus-stack-nodes-aix
+  labels:
+    grafana_dashboard: '1'
+    app: kube-prometheus-stack-grafana
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/instance: kube-prometheus-stack
+    app.kubernetes.io/part-of: kube-prometheus-stack
+    release: kube-prometheus-stack
+    heritage: Helm
+data:
+  nodes-aix.json: '{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  (1
+    - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\",
+    instance=\"$instance\", cluster=\"$cluster\"}[$__rate_interval])))\n/ ignoring(cpu)
+    group_left\n  count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\",
+    mode=\"idle\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n","format":"time_series","intervalFactor":5,"legendFormat":"{{cpu}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU
+    Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node_load1{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"1m
+    load average","refId":"A"},{"expr":"node_load5{job=\"node-exporter\", instance=\"$instance\",
+    cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"5m
+    load average","refId":"B"},{"expr":"node_load15{job=\"node-exporter\", instance=\"$instance\",
+    cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"15m
+    load average","refId":"C"},{"expr":"count(node_cpu_seconds_total{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\", mode=\"idle\"})","format":"time_series","intervalFactor":2,"legendFormat":"logical
+    cores","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Load
+    Average","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":false,"steppedLine":false,"targets":[{"expr":"node_memory_total_bytes{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"Physical
+    Memory","refId":"A"},{"expr":"(\n    node_memory_total_bytes{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\"} -\n    node_memory_available_bytes{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"Memory
+    Used","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory
+    Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50,
+    172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245,
+    54, 54, 0.9)","value":90}]},"unit":"percent"}},"gridPos":{},"id":5,"span":3,"targets":[{"expr":"100
+    -\n(\n  avg(node_memory_available_bytes{job=\"node-exporter\", instance=\"$instance\",
+    cluster=\"$cluster\"}) /\n  avg(node_memory_total_bytes{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\"})\n  * 100\n)\n","format":"time_series","intervalFactor":2,"legendFormat":""}],"title":"Memory
+    Usage","transparent":false,"type":"gauge"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/
+    read| written/","yaxis":1},{"alias":"/ io time/","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_disk_read_bytes_total{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}
+    read","refId":"A"},{"expr":"rate(node_disk_written_bytes_total{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}
+    written","refId":"B"},{"expr":"rate(node_disk_io_time_seconds_total{job=\"node-exporter\",
+    instance=\"$instance\", cluster=\"$cluster\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}
+    io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk
+    I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{},"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byName","options":"Mounted
+    on"},"properties":[{"id":"custom.width","value":260}]},{"matcher":{"id":"byName","options":"Size"},"properties":[{"id":"custom.width","value":93}]},{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":72}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"custom.width","value":88}]},{"matcher":{"id":"byName","options":"Used,
+    %"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.displayMode","value":"gradient-gauge"},{"id":"max","value":1},{"id":"min","value":0}]}]},"gridPos":{},"id":7,"span":6,"targets":[{"expr":"max
+    by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\",
+    cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""},{"expr":"max
+    by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\",
+    cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""}],"title":"Disk
+    Space Usage","transformations":[{"id":"groupBy","options":{"fields":{"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value
[Diff truncated by flux-local]