feat: support rule stop by error metrics (#2755)

Signed-off-by: yisaer <[email protected]>

internal/server/promMetrics/metrcis.go

@@ -59,12 +59,9 @@ func SetRuleStatusCountGauge(isRunning bool, count int) {
 	RuleStatusCountGauge.WithLabelValues(lbl).Set(float64(count))
 }
 
-func SetRuleStatus(ruleID string, isRunning bool) {
-	v := 0
-	if isRunning {
-		v = 1
-	}
-	RuleStatusGauge.WithLabelValues(ruleID).Set(float64(v))
+func SetRuleStatus(ruleID string, value int) {
+	v := float64(value)
+	RuleStatusGauge.WithLabelValues(ruleID).Set(v)
 }
 
 func RemoveRuleStatus(ruleID string) {

internal/server/server.go

@@ -349,19 +349,33 @@ func runScheduleRuleChecker(exit <-chan struct{}) {
 	runScheduleRuleCheckerByInterval(d, exit)
 }
 
+type RuleStatusMetricsValue int
+
+const (
+	RuleStoppedByError RuleStatusMetricsValue = -1
+	RuleStopped        RuleStatusMetricsValue = 0
+	RuleRunning        RuleStatusMetricsValue = 1
+)
+
 func handleAllRuleStatusMetrics(rs []ruleWrapper) {
 	if conf.Config != nil && conf.Config.Basic.Prometheus {
 		var runningCount int
 		var stopCount int
+		var v RuleStatusMetricsValue
 		for _, r := range rs {
 			id := r.rule.Id
-			isRunning := r.state == rule.RuleStarted
-			if isRunning {
+			switch r.state {
+			case rule.RuleStarted:
 				runningCount++
-			} else {
+				v = RuleRunning
+			case rule.RuleStopped, rule.RuleTerminated, rule.RuleWait:
+				stopCount++
+				v = RuleStopped
+			default:
 				stopCount++
+				v = RuleStoppedByError
 			}
-			promMetrics.SetRuleStatus(id, isRunning)
+			promMetrics.SetRuleStatus(id, int(v))
 		}
 		promMetrics.SetRuleStatusCountGauge(true, runningCount)
 		promMetrics.SetRuleStatusCountGauge(false, stopCount)