feat: canary_check_invalid_count metric (#2261)

* feat: canary_check_invalid_count metric * chore: undo some formatting changes
flanksource · Oct 25, 2024 · d85f05b · d85f05b
1 parent dbc85a6
commit d85f05b
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/checks/http.go b/checks/http.go
@@ -164,7 +164,7 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 
 	//nolint:staticcheck
 	if check.Endpoint != "" && check.URL != "" {
-		return results.Failf("cannot specify both endpoint and url")
+		return results.Invalidf("cannot specify both endpoint and url")
 	}
 
 	//nolint:staticcheck

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -62,6 +62,14 @@ func setupMetrics() {
 		checkLabels,
 	)
 
+	OpsInvalidCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "canary_check_invalid_count",
+			Help: "The total number of invalid checks",
+		},
+		[]string{"type", "endpoint", "canary_name", "canary_namespace", "owner", "severity", "key", "name"},
+	)
+
 	CanaryCheckInfo = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "canary_check_info",
@@ -86,7 +94,7 @@ func setupMetrics() {
 		checkLabels,
 	)
 
-	prometheus.MustRegister(Gauge, CanaryCheckInfo, OpsCount, OpsSuccessCount, OpsFailedCount, RequestLatency)
+	prometheus.MustRegister(Gauge, CanaryCheckInfo, OpsCount, OpsSuccessCount, OpsInvalidCount, OpsFailedCount, RequestLatency)
 }
 
 var (
@@ -103,6 +111,7 @@ var (
 	Gauge           *prometheus.GaugeVec
 
 	// Check specific metrics
+	OpsInvalidCount *prometheus.CounterVec
 	OpsCount        *prometheus.CounterVec
 	OpsFailedCount  *prometheus.CounterVec
 	OpsSuccessCount *prometheus.CounterVec
@@ -259,6 +268,12 @@ func Record(
 			}
 		}
 	} else {
+		if result.Invalid {
+			OpsInvalidCount.WithLabelValues(checkMetricLabels...).Inc()
+		} else {
+			OpsFailedCount.WithLabelValues(checkMetricLabels...).Inc()
+		}
+
 		fail.Append(1)
 		Gauge.WithLabelValues(gaugeLabels...).Set(1)
 
@@ -272,6 +287,7 @@ func Record(
 	} else {
 		_latency = types.Latency{}
 	}
+
 	return _uptime, _latency
 }