From f21c2ba4c0a9c3108209c1e59ec75d52e388235e Mon Sep 17 00:00:00 2001 From: Daniel Hrabovcak Date: Thu, 23 May 2024 12:24:35 -0400 Subject: [PATCH] feat: populate rules status --- ...onitoring.googleapis.com_clusterrules.yaml | 38 ++ ...monitoring.googleapis.com_globalrules.yaml | 38 ++ .../crds/monitoring.googleapis.com_rules.yaml | 38 ++ doc/api.md | 27 +- manifests/setup.yaml | 105 ++++++ .../apis/monitoring/v1/monitoring_types.go | 16 +- pkg/operator/apis/monitoring/v1/pod_types.go | 4 - .../apis/monitoring/v1/pod_types_test.go | 5 +- .../apis/monitoring/v1/rules_types.go | 14 +- pkg/operator/collection.go | 29 +- pkg/operator/collection_test.go | 6 +- pkg/operator/rules.go | 80 ++++- pkg/operator/rules_test.go | 337 ++++++++++++++++++ 13 files changed, 685 insertions(+), 52 deletions(-) diff --git a/charts/operator/crds/monitoring.googleapis.com_clusterrules.yaml b/charts/operator/crds/monitoring.googleapis.com_clusterrules.yaml index 3c48ff4a92..327e1fc777 100644 --- a/charts/operator/crds/monitoring.googleapis.com_clusterrules.yaml +++ b/charts/operator/crds/monitoring.googleapis.com_clusterrules.yaml @@ -124,6 +124,44 @@ spec: type: object status: description: Most recently observed status of the resource. + properties: + conditions: + description: Represents the latest available observations of a podmonitor's + current state. + items: + description: MonitoringCondition describes the condition of a PodMonitoring. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: MonitoringConditionType is the type of MonitoringCondition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: The generation observed by the controller. + format: int64 + type: integer type: object required: - spec diff --git a/charts/operator/crds/monitoring.googleapis.com_globalrules.yaml b/charts/operator/crds/monitoring.googleapis.com_globalrules.yaml index ee6566e0a0..4ae39f0490 100644 --- a/charts/operator/crds/monitoring.googleapis.com_globalrules.yaml +++ b/charts/operator/crds/monitoring.googleapis.com_globalrules.yaml @@ -123,6 +123,44 @@ spec: type: object status: description: Most recently observed status of the resource. + properties: + conditions: + description: Represents the latest available observations of a podmonitor's + current state. + items: + description: MonitoringCondition describes the condition of a PodMonitoring. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: MonitoringConditionType is the type of MonitoringCondition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: The generation observed by the controller. + format: int64 + type: integer type: object required: - spec diff --git a/charts/operator/crds/monitoring.googleapis.com_rules.yaml b/charts/operator/crds/monitoring.googleapis.com_rules.yaml index b293b8f520..4bff18b6fa 100644 --- a/charts/operator/crds/monitoring.googleapis.com_rules.yaml +++ b/charts/operator/crds/monitoring.googleapis.com_rules.yaml @@ -124,6 +124,44 @@ spec: type: object status: description: Most recently observed status of the resource. + properties: + conditions: + description: Represents the latest available observations of a podmonitor's + current state. + items: + description: MonitoringCondition describes the condition of a PodMonitoring. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: MonitoringConditionType is the type of MonitoringCondition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: The generation observed by the controller. + format: int64 + type: integer type: object required: - spec diff --git a/doc/api.md b/doc/api.md index 571f31e3b3..70a9cf1943 100644 --- a/doc/api.md +++ b/doc/api.md @@ -1385,7 +1385,7 @@ monitoring resource was created successfully.

-(Appears in: ClusterNodeMonitoring, PodMonitoringStatus) +(Appears in: ClusterNodeMonitoring, PodMonitoringStatus, RulesStatus)

MonitoringStatus holds status information of a monitoring resource.

@@ -2425,6 +2425,31 @@ RulesStatus

RulesStatus contains status information for a Rules resource.

+ + + + + + + + + + + + + +
FieldDescription
+MonitoringStatus
+ + +MonitoringStatus + + +		 +

+(Members of MonitoringStatus are embedded into this type.) +

+

SampleGroup diff --git a/manifests/setup.yaml b/manifests/setup.yaml index b827933ddb..44cf7445b6 100644 --- a/manifests/setup.yaml +++ b/manifests/setup.yaml @@ -1298,6 +1298,41 @@ spec: type: object status: description: Most recently observed status of the resource. + properties: + conditions: + description: Represents the latest available observations of a podmonitor's current state. + items: + description: MonitoringCondition describes the condition of a PodMonitoring. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: MonitoringConditionType is the type of MonitoringCondition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: The generation observed by the controller. + format: int64 + type: integer type: object required: - spec @@ -1520,6 +1555,41 @@ spec: type: object status: description: Most recently observed status of the resource. + properties: + conditions: + description: Represents the latest available observations of a podmonitor's current state. + items: + description: MonitoringCondition describes the condition of a PodMonitoring. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: MonitoringConditionType is the type of MonitoringCondition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: The generation observed by the controller. + format: int64 + type: integer type: object required: - spec @@ -3378,6 +3448,41 @@ spec: type: object status: description: Most recently observed status of the resource. + properties: + conditions: + description: Represents the latest available observations of a podmonitor's current state. + items: + description: MonitoringCondition describes the condition of a PodMonitoring. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: MonitoringConditionType is the type of MonitoringCondition. + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: The generation observed by the controller. + format: int64 + type: integer type: object required: - spec diff --git a/pkg/operator/apis/monitoring/v1/monitoring_types.go b/pkg/operator/apis/monitoring/v1/monitoring_types.go index ba4f4a6f36..82e0e7645b 100644 --- a/pkg/operator/apis/monitoring/v1/monitoring_types.go +++ b/pkg/operator/apis/monitoring/v1/monitoring_types.go @@ -68,6 +68,10 @@ func NewDefaultConditions(now metav1.Time) []MonitoringCondition { } } +func (cond *MonitoringCondition) IsValid() bool { + return cond.Type != "" && cond.Status != "" +} + // MonitoringStatus holds status information of a monitoring resource. type MonitoringStatus struct { // The generation observed by the controller. @@ -77,17 +81,17 @@ type MonitoringStatus struct { Conditions []MonitoringCondition `json:"conditions,omitempty"` } -// SetMonitoringCondition merges the provided condition if the resource generation changed or there is -// a status condition state transition. -func (status *MonitoringStatus) SetMonitoringCondition(gen int64, now metav1.Time, cond *MonitoringCondition) (bool, error) { +// SetMonitoringCondition merges the provided valid condition if the resource generation changed or +// there is a status condition state transition. +func (status *MonitoringStatus) SetMonitoringCondition(gen int64, now metav1.Time, cond *MonitoringCondition) bool { var ( specChanged = status.ObservedGeneration != gen statusTransition, update bool conds = make(map[MonitoringConditionType]*MonitoringCondition) ) - if cond.Type == "" || cond.Status == "" { - return update, errInvalidCond + if !cond.IsValid() { + return false } // Set up defaults. @@ -124,5 +128,5 @@ func (status *MonitoringStatus) SetMonitoringCondition(gen int64, now metav1.Tim } } - return update, nil + return update } diff --git a/pkg/operator/apis/monitoring/v1/pod_types.go b/pkg/operator/apis/monitoring/v1/pod_types.go index 33387c5996..031f46b7ca 100644 --- a/pkg/operator/apis/monitoring/v1/pod_types.go +++ b/pkg/operator/apis/monitoring/v1/pod_types.go @@ -26,10 +26,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -var ( - errInvalidCond = fmt.Errorf("condition needs both 'Type' and 'Status' fields set") -) - // PodMonitoringCRD represents a Kubernetes CRD that monitors Pod endpoints. type PodMonitoringCRD interface { MonitoringCRD diff --git a/pkg/operator/apis/monitoring/v1/pod_types_test.go b/pkg/operator/apis/monitoring/v1/pod_types_test.go index e39043a9cc..97d671523d 100644 --- a/pkg/operator/apis/monitoring/v1/pod_types_test.go +++ b/pkg/operator/apis/monitoring/v1/pod_types_test.go @@ -190,10 +190,7 @@ func TestSetMonitoringCondition(t *testing.T) { for _, c := range cases { t.Run(c.doc, func(t *testing.T) { got := c.curr - change, err := got.SetMonitoringCondition(c.generation, c.now, c.cond) - if err != nil { - t.Fatalf("set podmonitoring condition: %s", err) - } + change := got.SetMonitoringCondition(c.generation, c.now, c.cond) // Get resolved podmonitorings. if change != c.change { diff --git a/pkg/operator/apis/monitoring/v1/rules_types.go b/pkg/operator/apis/monitoring/v1/rules_types.go index c5de1bf7d3..5331e862ae 100644 --- a/pkg/operator/apis/monitoring/v1/rules_types.go +++ b/pkg/operator/apis/monitoring/v1/rules_types.go @@ -55,6 +55,10 @@ func (*Rules) ValidateDelete() (admission.Warnings, error) { return nil, nil } +func (r *Rules) GetMonitoringStatus() *MonitoringStatus { + return &r.Status.MonitoringStatus +} + // RulesList is a list of Rules. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type RulesList struct { @@ -100,6 +104,10 @@ func (*ClusterRules) ValidateDelete() (admission.Warnings, error) { return nil, nil } +func (r *ClusterRules) GetMonitoringStatus() *MonitoringStatus { + return &r.Status.MonitoringStatus +} + // ClusterRulesList is a list of ClusterRules. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type ClusterRulesList struct { @@ -144,6 +152,10 @@ func (*GlobalRules) ValidateDelete() (admission.Warnings, error) { return nil, nil } +func (r *GlobalRules) GetMonitoringStatus() *MonitoringStatus { + return &r.Status.MonitoringStatus +} + // GlobalRulesList is a list of GlobalRules. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type GlobalRulesList struct { @@ -192,5 +204,5 @@ type Rule struct { // RulesStatus contains status information for a Rules resource. type RulesStatus struct { - // TODO: add status information. + MonitoringStatus `json:",inline"` } diff --git a/pkg/operator/collection.go b/pkg/operator/collection.go index b61caf85ae..136dca2b80 100644 --- a/pkg/operator/collection.go +++ b/pkg/operator/collection.go @@ -153,7 +153,7 @@ func patchMonitoringStatus(ctx context.Context, kubeClient client.Client, obj cl patch := client.RawPatch(types.MergePatchType, patchBytes) if err := kubeClient.Status().Patch(ctx, obj, patch); err != nil { - return err + return fmt.Errorf("patch status: %w", err) } return nil } @@ -422,14 +422,7 @@ func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *mo } cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, cfgs...) - change, err := pmon.Status.SetMonitoringCondition(pmon.GetGeneration(), metav1.Now(), cond) - if err != nil { - // Log an error but let operator continue to avoid getting stuck - // on a potential bad resource. - logger.Error(err, "setting podmonitoring status state", "namespace", pmon.Namespace, "name", pmon.Name) - } - - if change { + if pmon.Status.SetMonitoringCondition(pmon.GetGeneration(), metav1.Now(), cond) { r.statusUpdates = append(r.statusUpdates, &pmon) } } @@ -463,14 +456,7 @@ func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *mo } cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, cfgs...) - change, err := cmon.Status.SetMonitoringCondition(cmon.GetGeneration(), metav1.Now(), cond) - if err != nil { - // Log an error but let operator continue to avoid getting stuck - // on a potential bad resource. - logger.Error(err, "setting clusterpodmonitoring status state", "namespace", cmon.Namespace, "name", cmon.Name) - } - - if change { + if cmon.Status.SetMonitoringCondition(cmon.GetGeneration(), metav1.Now(), cond) { r.statusUpdates = append(r.statusUpdates, &cmon) } } @@ -516,14 +502,7 @@ func (r *collectionReconciler) makeCollectorConfig(ctx context.Context, spec *mo } cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, cfgs...) - change, err := cm.Status.SetMonitoringCondition(cm.GetGeneration(), metav1.Now(), cond) - if err != nil { - // Log an error but let operator continue to avoid getting stuck - // on a potential bad resource. - logger.Error(err, "setting clusternodemonitoring status state", "namespace", cm.Namespace, "name", cm.Name) - } - - if change { + if cm.Status.SetMonitoringCondition(cm.GetGeneration(), metav1.Now(), cond) { r.statusUpdates = append(r.statusUpdates, &cm) } } diff --git a/pkg/operator/collection_test.go b/pkg/operator/collection_test.go index fbc4aba9d9..a1f3b9e24e 100644 --- a/pkg/operator/collection_test.go +++ b/pkg/operator/collection_test.go @@ -56,7 +56,11 @@ func newFakeClientBuilder() *fake.ClientBuilder { return fake.NewClientBuilder(). WithScheme(testScheme). WithStatusSubresource(&monitoringv1.PodMonitoring{}). - WithStatusSubresource(&monitoringv1.ClusterPodMonitoring{}) + WithStatusSubresource(&monitoringv1.ClusterPodMonitoring{}). + WithStatusSubresource(&monitoringv1.ClusterNodeMonitoring{}). + WithStatusSubresource(&monitoringv1.Rules{}). + WithStatusSubresource(&monitoringv1.ClusterRules{}). + WithStatusSubresource(&monitoringv1.GlobalRules{}) } // Tests that the collection does not overwrite the non-managed status fields. diff --git a/pkg/operator/rules.go b/pkg/operator/rules.go index 073ac40e5f..407282489f 100644 --- a/pkg/operator/rules.go +++ b/pkg/operator/rules.go @@ -16,6 +16,7 @@ package operator import ( "context" + "errors" "fmt" "github.com/go-logr/logr" @@ -202,6 +203,7 @@ func hasGlobalRules(ctx context.Context, c client.Client) (bool, error) { return len(rules.Items) > 0, nil } +// ensureRuleConfigs updates the Prometheus Rules ConfigMap. func (r *rulesReconciler) ensureRuleConfigs(ctx context.Context, projectID, location, cluster string, compression monitoringv1.CompressionType) error { logger, _ := logr.FromContext(ctx) @@ -234,48 +236,98 @@ func (r *rulesReconciler) ensureRuleConfigs(ctx context.Context, projectID, loca if err := r.client.List(ctx, &rulesList); err != nil { return fmt.Errorf("list rules: %w", err) } - for _, rs := range rulesList.Items { + + now := metav1.Now() + conditionSuccess := &monitoringv1.MonitoringCondition{ + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionTrue, + } + var statusUpdates []monitoringv1.MonitoringCRD + + for i := range rulesList.Items { + rs := &rulesList.Items[i] result, err := rs.RuleGroupsConfig(projectID, location, cluster) if err != nil { - // TODO(freinartz): update resource condition. - logger.Error(err, "converting rules failed", "rules_namespace", rs.Namespace, "rules_name", rs.Name) + msg := "generating rule config failed" + if rs.Status.SetMonitoringCondition(rs.GetGeneration(), now, &monitoringv1.MonitoringCondition{ + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + Message: msg, + Reason: err.Error(), + }) { + statusUpdates = append(statusUpdates, rs) + } + logger.Error(err, "convert rules", "err", err, "namespace", rs.Namespace, "name", rs.Name) + continue } filename := fmt.Sprintf("rules__%s__%s.yaml", rs.Namespace, rs.Name) if err := setConfigMapData(cm, compression, filename, result); err != nil { return err } + + if rs.Status.SetMonitoringCondition(rs.GetGeneration(), now, conditionSuccess) { + statusUpdates = append(statusUpdates, rs) + } } var clusterRulesList monitoringv1.ClusterRulesList if err := r.client.List(ctx, &clusterRulesList); err != nil { return fmt.Errorf("list cluster rules: %w", err) } - for _, rs := range clusterRulesList.Items { + for i := range clusterRulesList.Items { + rs := &clusterRulesList.Items[i] result, err := rs.RuleGroupsConfig(projectID, location, cluster) if err != nil { - // TODO(freinartz): update resource condition. - logger.Error(err, "converting rules failed", "clusterrules_name", rs.Name) + msg := "generating rule config failed" + if rs.Status.SetMonitoringCondition(rs.Generation, now, &monitoringv1.MonitoringCondition{ + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + Message: msg, + Reason: err.Error(), + }) { + statusUpdates = append(statusUpdates, rs) + } + logger.Error(err, "convert rules", "err", err, "namespace", rs.Namespace, "name", rs.Name) + continue } filename := fmt.Sprintf("clusterrules__%s.yaml", rs.Name) if err := setConfigMapData(cm, compression, filename, result); err != nil { return err } + + if rs.Status.SetMonitoringCondition(rs.GetGeneration(), now, conditionSuccess) { + statusUpdates = append(statusUpdates, rs) + } } var globalRulesList monitoringv1.GlobalRulesList if err := r.client.List(ctx, &globalRulesList); err != nil { return fmt.Errorf("list global rules: %w", err) } - for _, rs := range globalRulesList.Items { + for i := range globalRulesList.Items { + rs := &globalRulesList.Items[i] result, err := rs.RuleGroupsConfig() if err != nil { - // TODO(freinartz): update resource condition. - logger.Error(err, "converting rules failed", "globalrules_name", rs.Name) + msg := "generating rule config failed" + if rs.Status.SetMonitoringCondition(rs.Generation, now, &monitoringv1.MonitoringCondition{ + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + Message: msg, + Reason: err.Error(), + }) { + statusUpdates = append(statusUpdates, rs) + } + logger.Error(err, "convert rules", "err", err, "namespace", rs.Namespace, "name", rs.Name) + continue } filename := fmt.Sprintf("globalrules__%s.yaml", rs.Name) if err := setConfigMapData(cm, compression, filename, result); err != nil { return err } + + if rs.Status.SetMonitoringCondition(rs.GetGeneration(), now, conditionSuccess) { + statusUpdates = append(statusUpdates, rs) + } } // Create or update generated rule ConfigMap. @@ -286,5 +338,13 @@ func (r *rulesReconciler) ensureRuleConfigs(ctx context.Context, projectID, loca } else if err != nil { return fmt.Errorf("update generated rules: %w", err) } - return nil + + var errs []error + for _, obj := range statusUpdates { + if err := patchMonitoringStatus(ctx, r.client, obj, obj.GetMonitoringStatus()); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) } diff --git a/pkg/operator/rules_test.go b/pkg/operator/rules_test.go index a82e7bae0b..1dbb2f6b36 100644 --- a/pkg/operator/rules_test.go +++ b/pkg/operator/rules_test.go @@ -17,10 +17,14 @@ package operator import ( "context" "errors" + "fmt" "testing" + "time" monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1" + "github.com/google/go-cmp/cmp" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -59,6 +63,339 @@ func TestHasRules(t *testing.T) { } } +func TestRulesStatus(t *testing.T) { + // The fake client truncates seconds, so create a time that can't be rounded down. + timeDefault := metav1.NewTime(time.Date(2024, 5, 23, 1, 23, 0, 0, time.UTC)) + timeAfter := metav1.NewTime(timeDefault.Add(time.Minute)) + var testCases []struct { + obj monitoringv1.MonitoringCRD + expectedStatus monitoringv1.MonitoringStatus + } + + addTestCases := func(name string, newObj func(objectMeta metav1.ObjectMeta, spec monitoringv1.RulesSpec, status monitoringv1.RulesStatus) monitoringv1.MonitoringCRD) { + testCases = append(testCases, []struct { + obj monitoringv1.MonitoringCRD + expectedStatus monitoringv1.MonitoringStatus + }{ + { + obj: newObj( + metav1.ObjectMeta{ + Name: fmt.Sprintf("invalid-%s-no-condition", name), + Generation: 2, + }, + monitoringv1.RulesSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Rules: []monitoringv1.Rule{ + { + Record: "test_record", + Expr: "test_expr{", + }, + }, + }, + }, + }, + monitoringv1.RulesStatus{}, + ), + expectedStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 2, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + LastUpdateTime: timeAfter, + LastTransitionTime: timeAfter, + Message: "generating rule config failed", + }, + }, + }, + }, + { + obj: newObj( + metav1.ObjectMeta{ + Name: fmt.Sprintf("invalid-%s-outdated-condition", name), + Generation: 2, + }, + monitoringv1.RulesSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Rules: []monitoringv1.Rule{ + { + Record: "test_record", + Expr: "test_expr{", + }, + }, + }, + }, + }, + monitoringv1.RulesStatus{ + MonitoringStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 1, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionTrue, + LastUpdateTime: timeDefault, + LastTransitionTime: timeDefault, + }, + }, + }, + }, + ), + expectedStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 2, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + LastUpdateTime: timeAfter, + LastTransitionTime: timeAfter, + Message: "generating rule config failed", + }, + }, + }, + }, + { + obj: newObj( + metav1.ObjectMeta{ + Name: fmt.Sprintf("invalid-%s-correct-condition", name), + Generation: 2, + }, + monitoringv1.RulesSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Rules: []monitoringv1.Rule{ + { + Record: "test_record", + Expr: "test_expr{", + }, + }, + }, + }, + }, + monitoringv1.RulesStatus{ + MonitoringStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 1, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + LastUpdateTime: timeDefault, + LastTransitionTime: timeDefault, + }, + }, + }, + }, + ), + expectedStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 2, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + LastUpdateTime: timeAfter, + LastTransitionTime: timeDefault, + Message: "generating rule config failed", + }, + }, + }, + }, + { + obj: newObj( + metav1.ObjectMeta{ + Name: fmt.Sprintf("valid-%s-no-condition", name), + Generation: 1, + }, + monitoringv1.RulesSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Rules: []monitoringv1.Rule{ + { + Record: "test_record", + Expr: "test_expr", + }, + }, + }, + }, + }, + monitoringv1.RulesStatus{}, + ), + expectedStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 1, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionTrue, + LastUpdateTime: timeAfter, + LastTransitionTime: timeAfter, + }, + }, + }, + }, + { + obj: newObj( + metav1.ObjectMeta{ + Name: fmt.Sprintf("valid-%s-outdated-condition", name), + Generation: 2, + }, + monitoringv1.RulesSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Rules: []monitoringv1.Rule{ + { + Record: "test_record", + Expr: "test_expr", + }, + }, + }, + }, + }, + monitoringv1.RulesStatus{ + MonitoringStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 1, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionFalse, + LastUpdateTime: timeDefault, + LastTransitionTime: timeDefault, + }, + }, + }, + }, + ), + expectedStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 2, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionTrue, + LastUpdateTime: timeAfter, + LastTransitionTime: timeAfter, + }, + }, + }, + }, + { + obj: newObj( + metav1.ObjectMeta{ + Name: fmt.Sprintf("valid-%s-correct-condition", name), + Generation: 2, + }, + monitoringv1.RulesSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Rules: []monitoringv1.Rule{ + { + Record: "test_record", + Expr: "test_expr", + }, + }, + }, + }, + }, + monitoringv1.RulesStatus{ + MonitoringStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 1, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionTrue, + LastUpdateTime: timeDefault, + LastTransitionTime: timeDefault, + }, + }, + }, + }, + ), + expectedStatus: monitoringv1.MonitoringStatus{ + ObservedGeneration: 2, + Conditions: []monitoringv1.MonitoringCondition{ + { + Type: monitoringv1.ConfigurationCreateSuccess, + Status: corev1.ConditionTrue, + LastUpdateTime: timeAfter, + LastTransitionTime: timeDefault, + }, + }, + }, + }, + }...) + } + addTestCases("namespaced-rule", func(objectMeta metav1.ObjectMeta, spec monitoringv1.RulesSpec, status monitoringv1.RulesStatus) monitoringv1.MonitoringCRD { + return &monitoringv1.Rules{ + ObjectMeta: objectMeta, + Spec: spec, + Status: status, + } + }) + addTestCases("cluster-rule", func(objectMeta metav1.ObjectMeta, spec monitoringv1.RulesSpec, status monitoringv1.RulesStatus) monitoringv1.MonitoringCRD { + return &monitoringv1.ClusterRules{ + ObjectMeta: objectMeta, + Spec: spec, + Status: status, + } + }) + addTestCases("global-rule", func(objectMeta metav1.ObjectMeta, spec monitoringv1.RulesSpec, status monitoringv1.RulesStatus) monitoringv1.MonitoringCRD { + return &monitoringv1.GlobalRules{ + ObjectMeta: objectMeta, + Spec: spec, + Status: status, + } + }) + + objs := make([]client.Object, 0, len(testCases)) + for _, tc := range testCases { + objs = append(objs, tc.obj) + } + kubeClient := newFakeClientBuilder(). + WithObjects(objs...). + Build() + + ctx := context.Background() + r := rulesReconciler{ + client: kubeClient, + } + + if err := r.ensureRuleConfigs(ctx, "", "", "", monitoringv1.CompressionNone); err != nil { + t.Fatal("ensure rules configs:", err) + } + + for _, tc := range testCases { + t.Run(tc.obj.GetName(), func(t *testing.T) { + objectKey := client.ObjectKeyFromObject(tc.obj) + if err := kubeClient.Get(ctx, objectKey, tc.obj); err != nil { + t.Fatal("get obj:", err) + } + + status := tc.obj.GetMonitoringStatus() + if len(status.Conditions) != 1 { + t.Fatalf("invalid %q conditions amount, expected 1 but got %d", objectKey, len(status.Conditions)) + } + condition := &status.Conditions[0] + // If time changed, normalize to the "after time", since we don't mock the process time. + if !condition.LastTransitionTime.Equal(&timeDefault) { + condition.LastTransitionTime = timeAfter + } + if !condition.LastUpdateTime.Equal(&timeDefault) { + condition.LastUpdateTime = timeAfter + } + // The message is good enough. Don't need reason. + condition.Reason = "" + + if diff := cmp.Diff(&tc.expectedStatus, status); diff != "" { + t.Errorf("expected %q condition (-want, +got): %s", objectKey, diff) + } + }) + } +} + func TestScaleRuleConsumers(t *testing.T) { var alertmanagerReplicas int32 alertManager := appsv1.StatefulSet{