Merge pull request #879 from pyrra-dev/alerting-burnrates-absent

Add burnrate and absent alerting configuration
pyrra-dev · Aug 28, 2023 · f618d80 · f618d80
2 parents 3c84c19 + 0dce672
commit f618d80
Show file tree

Hide file tree

Showing 8 changed files with 228 additions and 126 deletions.
diff --git a/config/crd/bases/pyrra.dev_servicelevelobjectives.json b/config/crd/bases/pyrra.dev_servicelevelobjectives.json
@@ -66,6 +66,14 @@
                   "alerting": {
                     "description": "Alerting customizes the alerting rules generated by Pyrra.",
                     "properties": {
+                      "absent": {
+                        "default": true,
+                        "type": "boolean"
+                      },
+                      "burnrates": {
+                        "default": true,
+                        "type": "boolean"
+                      },
                       "disabled": {
                         "description": "Disabled is used to disable the generation of alerts. Recording rules are still generated.",
                         "type": "boolean"

diff --git a/config/crd/bases/pyrra.dev_servicelevelobjectives.yaml b/config/crd/bases/pyrra.dev_servicelevelobjectives.yaml
@@ -54,6 +54,12 @@ spec:
               alerting:
                 description: Alerting customizes the alerting rules generated by Pyrra.
                 properties:
+                  absent:
+                    default: true
+                    type: boolean
+                  burnrates:
+                    default: true
+                    type: boolean
                   disabled:
                     description: Disabled is used to disable the generation of alerts.
                       Recording rules are still generated.

diff --git a/kubernetes/api/v1alpha1/servicelevelobjective_types.go b/kubernetes/api/v1alpha1/servicelevelobjective_types.go
@@ -115,9 +115,18 @@ type ServiceLevelIndicator struct {
 
 type Alerting struct {
 	// +optional
+	// +kubebuilder:deprecatedversion:warning="disabled is deprecated. Use burnrates instead."
 	// Disabled is used to disable the generation of alerts. Recording rules are still generated.
 	Disabled *bool `json:"disabled,omitempty"`
 
+	// +optional
+	// +kubebuilder:default:=true
+	Burnrates *bool `json:"burnrates,omitempty"`
+
+	// +optional
+	// +kubebuilder:default:=true
+	Absent *bool `json:"absent,omitempty"`
+
 	// +optional
 	// Name is used as the name of the alert generated by Pyrra. Defaults to "ErrorBudgetBurn".
 	Name string `json:"name,omitempty"`
@@ -347,6 +356,14 @@ func (in *ServiceLevelObjective) Internal() (slo.Objective, error) {
 	if in.Spec.Alerting.Disabled != nil {
 		alerting.Disabled = *in.Spec.Alerting.Disabled
 	}
+	alerting.Burnrates = true
+	if in.Spec.Alerting.Burnrates != nil {
+		alerting.Burnrates = *in.Spec.Alerting.Burnrates
+	}
+	alerting.Absent = true
+	if in.Spec.Alerting.Absent != nil {
+		alerting.Absent = *in.Spec.Alerting.Absent
+	}
 
 	if in.Spec.Alerting.Name != "" {
 		alerting.Name = in.Spec.Alerting.Name

diff --git a/kubernetes/api/v1alpha1/servicelevelobjective_types_test.go b/kubernetes/api/v1alpha1/servicelevelobjective_types_test.go
@@ -52,6 +52,10 @@ spec:
 			Description: "",
 			Target:      0.99,
 			Window:      model.Duration(7 * 24 * time.Hour),
+			Alerting: slo.Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: slo.Indicator{
 				Ratio: &slo.RatioIndicator{
 					Errors: slo.Metric{
@@ -101,6 +105,10 @@ spec:
 			Description: "",
 			Target:      0.9990000000000001, // TODO fix this? maybe not /100?
 			Window:      model.Duration(7 * 24 * time.Hour),
+			Alerting: slo.Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: slo.Indicator{
 				Ratio: &slo.RatioIndicator{
 					Errors: slo.Metric{
@@ -153,6 +161,10 @@ spec:
 			),
 			Target: 0.995,
 			Window: model.Duration(28 * 24 * time.Hour),
+			Alerting: slo.Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: slo.Indicator{
 				Latency: &slo.LatencyIndicator{
 					Success: slo.Metric{
@@ -203,6 +215,10 @@ spec:
 			),
 			Target: 0.995,
 			Window: model.Duration(7 * 24 * time.Hour),
+			Alerting: slo.Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: slo.Indicator{
 				Latency: &slo.LatencyIndicator{
 					Success: slo.Metric{
@@ -255,6 +271,10 @@ spec:
 			),
 			Target: 0.99,
 			Window: model.Duration(28 * 24 * time.Hour),
+			Alerting: slo.Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: slo.Indicator{
 				Latency: &slo.LatencyIndicator{
 					Success: slo.Metric{
@@ -306,6 +326,10 @@ spec:
 			),
 			Target: 0.99,
 			Window: model.Duration(14 * 24 * time.Hour),
+			Alerting: slo.Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: slo.Indicator{
 				Ratio: &slo.RatioIndicator{
 					Errors: slo.Metric{

diff --git a/kubernetes/api/v1alpha1/zz_generated.deepcopy.go b/kubernetes/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/slo/promql_test.go b/slo/promql_test.go
@@ -17,6 +17,10 @@ var (
 			Labels: labels.FromStrings(labels.MetricName, "monitoring-http-errors"),
 			Target: 0.99,
 			Window: model.Duration(28 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Ratio: &RatioIndicator{
 					Errors: Metric{
@@ -60,6 +64,10 @@ var (
 			Description: "",
 			Target:      0.999,
 			Window:      model.Duration(28 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Ratio: &RatioIndicator{
 					Errors: Metric{
@@ -95,6 +103,10 @@ var (
 			Labels: labels.FromStrings(labels.MetricName, "monitoring-http-latency"),
 			Target: 0.995,
 			Window: model.Duration(28 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Latency: &LatencyIndicator{
 					Success: Metric{
@@ -156,6 +168,10 @@ var (
 			Labels: labels.FromStrings(labels.MetricName, "monitoring-grpc-latency"),
 			Target: 0.995,
 			Window: model.Duration(7 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Latency: &LatencyIndicator{
 					Success: Metric{
@@ -191,6 +207,10 @@ var (
 			Labels: labels.FromStrings(labels.MetricName, "monitoring-prometheus-operator-errors"),
 			Target: 0.99,
 			Window: model.Duration(14 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Ratio: &RatioIndicator{
 					Errors: Metric{
@@ -219,6 +239,10 @@ var (
 			Labels: labels.FromStrings(labels.MetricName, "apiserver-write-response-errors"),
 			Target: 0.99,
 			Window: model.Duration(14 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Ratio: &RatioIndicator{
 					Errors: Metric{
@@ -243,13 +267,19 @@ var (
 	objectiveAPIServerRatioAlertingDisabled = func() Objective {
 		o := objectiveAPIServerRatio()
 		o.Alerting.Disabled = true
+		o.Alerting.Burnrates = false
+		o.Alerting.Absent = false
 		return o
 	}
 	objectiveAPIServerLatency = func() Objective {
 		return Objective{
 			Labels: labels.FromStrings(labels.MetricName, "apiserver-read-resource-latency"),
 			Target: 0.99,
 			Window: model.Duration(14 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				Latency: &LatencyIndicator{
 					Grouping: []string{
@@ -280,6 +310,8 @@ var (
 	objectiveAPIServerLatencyAlertingDisabled = func() Objective {
 		o := objectiveAPIServerLatency()
 		o.Alerting.Disabled = true
+		o.Alerting.Burnrates = false
+		o.Alerting.Absent = false
 		return o
 	}
 	objectiveAPIServerLatencyCustomAlertname = func() Objective {
@@ -292,6 +324,10 @@ var (
 			Labels: labels.FromStrings(labels.MetricName, "up-targets"),
 			Target: 0.99,
 			Window: model.Duration(28 * 24 * time.Hour),
+			Alerting: Alerting{
+				Burnrates: true,
+				Absent:    true,
+			},
 			Indicator: Indicator{
 				BoolGauge: &BoolGaugeIndicator{
 					Metric: Metric{Name: "up"},