Increasing the threshold for a file lag and reducing the severity to …

…warning Signed-off-by: Edward Welch <[email protected]>
holidayextras · Feb 26, 2020 · 8df56ce · 8df56ce
1 parent f9e37e2
commit 8df56ce
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/production/promtail-mixin/alerts.libsonnet b/production/promtail-mixin/alerts.libsonnet
@@ -40,15 +40,15 @@
           {
             alert: 'PromtailFileLagging',
             expr: |||
-              abs(promtail_file_bytes_total - promtail_read_bytes_total) > 100000
+              abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6
             |||,
             'for': '15m',
             labels: {
-              severity: 'critical',
+              severity: 'warning',
             },
             annotations: {
               message: |||
-                {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 100kb for more than 15m.
+                {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.
               |||,
             },
           },