From 302e9ef302b690a946eea21556802c40161655e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20G=C3=B3rski?= Date: Thu, 3 Aug 2023 11:20:58 +0200 Subject: [PATCH] feat: add extra-filter-labels param --- cmd/sloth/commands/generate.go | 16 ++++-- cmd/sloth/commands/k8scontroller.go | 20 ++++--- cmd/sloth/commands/validate.go | 14 +++-- .../helm/sloth/templates/deployment.yaml | 3 ++ .../helm/sloth/tests/values_test.go | 4 ++ deploy/kubernetes/helm/sloth/values.yaml | 1 + internal/app/generate/noop.go | 4 +- internal/app/generate/prometheus.go | 17 +++--- internal/app/kubecontroller/handler.go | 22 +++++--- internal/prometheus/recording_rules.go | 52 +++++++++++-------- internal/prometheus/recording_rules_test.go | 6 +-- 11 files changed, 104 insertions(+), 55 deletions(-) diff --git a/cmd/sloth/commands/generate.go b/cmd/sloth/commands/generate.go index 2d5c24ae..a944e5f6 100644 --- a/cmd/sloth/commands/generate.go +++ b/cmd/sloth/commands/generate.go @@ -36,6 +36,7 @@ type generateCommand struct { disableAlerts bool disableOptimizedRules bool extraLabels map[string]string + extraFilterLabels map[string]string sliPluginsPaths []string sloPeriodWindowsPath string sloPeriod string @@ -43,7 +44,10 @@ type generateCommand struct { // NewGenerateCommand returns the generate command. func NewGenerateCommand(app *kingpin.Application) Command { - c := &generateCommand{extraLabels: map[string]string{}} + c := &generateCommand{ + extraLabels: map[string]string{}, + extraFilterLabels: map[string]string{}, + } cmd := app.Command("generate", "Generates Prometheus SLOs.") cmd.Flag("input", "SLO spec input file path or directory (if directory is used, slos will be discovered recursively and out must be a directory).").Short('i').StringVar(&c.slosInput) cmd.Flag("out", "Generated rules output file path or directory. If `-` it will use stdout (if input is a directory this must be a directory).").Default("-").Short('o').StringVar(&c.slosOut) @@ -51,6 +55,7 @@ func NewGenerateCommand(app *kingpin.Application) Command { cmd.Flag("fs-include", "Filter regex to include matched discovered SLO file paths, everything else will be ignored. Exclude has preference (used with directory based input/output).").Short('n').StringVar(&c.slosIncludeRegex) cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels) + cmd.Flag("extra-filter-labels", "Extra labels that will be added to all the generated Prometheus rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. ('key=value' form, can be repeated).").StringMapVar(&c.extraFilterLabels) cmd.Flag("disable-recordings", "Disables recording rules generation.").BoolVar(&c.disableRecordings) cmd.Flag("disable-alerts", "Disables alert rules generation.").BoolVar(&c.disableAlerts) cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths) @@ -245,6 +250,7 @@ func (g generateCommand) Run(ctx context.Context, config RootConfig) error { disableAlerts: g.disableAlerts, disableOptimizedRules: g.disableOptimizedRules, extraLabels: g.extraLabels, + extraFilterLabels: g.extraFilterLabels, } for _, genTarget := range genTargets { @@ -305,6 +311,7 @@ type generator struct { disableAlerts bool disableOptimizedRules bool extraLabels map[string]string + extraFilterLabels map[string]string } // GeneratePrometheus generates the SLOs based on a raw regular Prometheus spec format input and outs a Prometheus raw yaml. @@ -433,9 +440,10 @@ func (g generator) generateRules(ctx context.Context, info info.Info, slos prome } result, err := controller.Generate(ctx, generate.Request{ - ExtraLabels: g.extraLabels, - Info: info, - SLOGroup: slos, + ExtraLabels: g.extraLabels, + ExtraFilterLabels: g.extraFilterLabels, + Info: info, + SLOGroup: slos, }) if err != nil { return nil, fmt.Errorf("could not generate prometheus rules: %w", err) diff --git a/cmd/sloth/commands/k8scontroller.go b/cmd/sloth/commands/k8scontroller.go index 5b0ffbff..a383f69d 100644 --- a/cmd/sloth/commands/k8scontroller.go +++ b/cmd/sloth/commands/k8scontroller.go @@ -53,6 +53,7 @@ const ( type kubeControllerCommand struct { extraLabels map[string]string + extraFilterLabels map[string]string workers int kubeConfig string kubeContext string @@ -73,7 +74,10 @@ type kubeControllerCommand struct { // NewKubeControllerCommand returns the Kubernetes controller command. func NewKubeControllerCommand(app *kingpin.Application) Command { - c := &kubeControllerCommand{extraLabels: map[string]string{}} + c := &kubeControllerCommand{ + extraLabels: map[string]string{}, + extraFilterLabels: map[string]string{}, + } cmd := app.Command("kubernetes-controller", "Runs Sloth in Kubernetes controller/operator mode.") cmd.Alias("controller") cmd.Alias("k8s-controller") @@ -92,6 +96,7 @@ func NewKubeControllerCommand(app *kingpin.Application) Command { cmd.Flag("hot-reload-addr", "The listen address for hot-reloading components that allow it.").Default(":8082").StringVar(&c.hotReloadAddr) cmd.Flag("hot-reload-path", "The webhook path for hot-reloading components that allow it.").Default("/-/reload").StringVar(&c.hotReloadPath) cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels) + cmd.Flag("extra-filter-labels", "Extra labels that will be added to all the generated Prometheus rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. ('key=value' form, can be repeated).").StringMapVar(&c.extraFilterLabels) cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths) cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath) cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod) @@ -318,12 +323,13 @@ func (k kubeControllerCommand) Run(ctx context.Context, config RootConfig) error // Create handler. config := kubecontroller.HandlerConfig{ - Generator: generator, - SpecLoader: k8sprometheus.NewCRSpecLoader(pluginRepo, sloPeriod), - Repository: k8sprometheus.NewPrometheusOperatorCRDRepo(ksvc, logger), - KubeStatusStorer: ksvc, - ExtraLabels: k.extraLabels, - Logger: logger, + Generator: generator, + SpecLoader: k8sprometheus.NewCRSpecLoader(pluginRepo, sloPeriod), + Repository: k8sprometheus.NewPrometheusOperatorCRDRepo(ksvc, logger), + KubeStatusStorer: ksvc, + ExtraLabels: k.extraLabels, + ExtraFilterLabels: k.extraFilterLabels, + Logger: logger, } handler, err := kubecontroller.NewHandler(config) if err != nil { diff --git a/cmd/sloth/commands/validate.go b/cmd/sloth/commands/validate.go index 1dc68cf3..25f86334 100644 --- a/cmd/sloth/commands/validate.go +++ b/cmd/sloth/commands/validate.go @@ -24,6 +24,7 @@ type validateCommand struct { slosExcludeRegex string slosIncludeRegex string extraLabels map[string]string + extraFilterLabels map[string]string sliPluginsPaths []string sloPeriodWindowsPath string sloPeriod string @@ -31,12 +32,16 @@ type validateCommand struct { // NewValidateCommand returns the validate command. func NewValidateCommand(app *kingpin.Application) Command { - c := &validateCommand{extraLabels: map[string]string{}} + c := &validateCommand{ + extraLabels: map[string]string{}, + extraFilterLabels: map[string]string{}, + } cmd := app.Command("validate", "Validates the SLO manifests and generation of Prometheus SLOs.") cmd.Flag("input", "SLO spec discovery path, will discover recursively all YAML files.").Short('i').Required().StringVar(&c.slosInput) cmd.Flag("fs-exclude", "Filter regex to ignore matched discovered SLO file paths.").Short('e').StringVar(&c.slosExcludeRegex) cmd.Flag("fs-include", "Filter regex to include matched discovered SLO file paths, everything else will be ignored. Exclude has preference.").Short('n').StringVar(&c.slosIncludeRegex) cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels) + cmd.Flag("extra-filter-labels", "Extra labels that will be added to all the generated Prometheus rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. ('key=value' form, can be repeated).").StringMapVar(&c.extraFilterLabels) cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths) cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath) cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod) @@ -126,9 +131,10 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { splittedSLOsData := splitYAML(slxData) gen := generator{ - logger: log.Noop, - windowsRepo: windowsRepo, - extraLabels: v.extraLabels, + logger: log.Noop, + windowsRepo: windowsRepo, + extraLabels: v.extraLabels, + extraFilterLabels: v.extraFilterLabels, } // Prepare file validation result and start validation result for every SLO in the file. diff --git a/deploy/kubernetes/helm/sloth/templates/deployment.yaml b/deploy/kubernetes/helm/sloth/templates/deployment.yaml index f4b75491..b1de7ae6 100644 --- a/deploy/kubernetes/helm/sloth/templates/deployment.yaml +++ b/deploy/kubernetes/helm/sloth/templates/deployment.yaml @@ -47,6 +47,9 @@ spec: {{- range $key, $val := .Values.sloth.extraLabels }} - --extra-labels={{ $key }}={{ $val }} {{- end}} + {{- range $key, $val := .Values.sloth.extraFilterLabels }} + - --extra-filter-labels={{ $key }}={{ $val }} + {{- end}} {{- if .Values.commonPlugins.enabled }} - --sli-plugins-path=/plugins {{- end }} diff --git a/deploy/kubernetes/helm/sloth/tests/values_test.go b/deploy/kubernetes/helm/sloth/tests/values_test.go index bbfcc51e..20f93fb6 100644 --- a/deploy/kubernetes/helm/sloth/tests/values_test.go +++ b/deploy/kubernetes/helm/sloth/tests/values_test.go @@ -27,6 +27,10 @@ func customValues() msi { "k1": "v1", "k2": "v2", }, + "extraFilterLabels": msi{ + "k3": "v3", + "k4": "v4", + }, }, "commonPlugins": msi{ diff --git a/deploy/kubernetes/helm/sloth/values.yaml b/deploy/kubernetes/helm/sloth/values.yaml index 8c8b309d..8f32ffc6 100644 --- a/deploy/kubernetes/helm/sloth/values.yaml +++ b/deploy/kubernetes/helm/sloth/values.yaml @@ -23,6 +23,7 @@ sloth: labelSelector: "" # Sloth will handle only the ones that match the selector. namespace: "" # The namespace where sloth will the CRs to process. extraLabels: {} # Labels that will be added to all the generated SLO Rules. + extraFilterLabels: {} # Labels that will be added to all the generated SLO Rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. defaultSloPeriod: "" # The slo period used by sloth (e.g. 30d). optimizedRules: true # Reduce prom load for calculating period window burnrates. debug: diff --git a/internal/app/generate/noop.go b/internal/app/generate/noop.go index c0ee12dd..af349fe5 100644 --- a/internal/app/generate/noop.go +++ b/internal/app/generate/noop.go @@ -13,7 +13,7 @@ type noopSLIRecordingRulesGenerator bool const NoopSLIRecordingRulesGenerator = noopSLIRecordingRulesGenerator(false) -func (noopSLIRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) { +func (noopSLIRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) { return nil, nil } @@ -21,7 +21,7 @@ type noopMetadataRecordingRulesGenerator bool const NoopMetadataRecordingRulesGenerator = noopMetadataRecordingRulesGenerator(false) -func (noopMetadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) { +func (noopMetadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) { return nil, nil } diff --git a/internal/app/generate/prometheus.go b/internal/app/generate/prometheus.go index 7bf2c2cf..cab1bc39 100644 --- a/internal/app/generate/prometheus.go +++ b/internal/app/generate/prometheus.go @@ -53,12 +53,12 @@ type AlertGenerator interface { // SLIRecordingRulesGenerator knows how to generate SLI recording rules. type SLIRecordingRulesGenerator interface { - GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) + GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) } // MetadataRecordingRulesGenerator knows how to generate metadata recording rules. type MetadataRecordingRulesGenerator interface { - GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) + GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) } // SLOAlertRulesGenerator knows hot to generate SLO alert rules. @@ -96,6 +96,9 @@ type Request struct { Info info.Info // ExtraLabels are the extra labels added to the SLOs on execution time. ExtraLabels map[string]string + // ExtraFilterLabels are the extra labels added to the SLOs on execution time and also will be added as filters to the generated recording rules. + // Can be used as templates in queries provided by the user. + ExtraFilterLabels map[string]string // SLOGroup are the SLOs group that will be used to generate the SLO results and Prom rules. SLOGroup prometheus.SLOGroup } @@ -120,10 +123,10 @@ func (s Service) Generate(ctx context.Context, r Request) (*Response, error) { results := make([]SLOResult, 0, len(r.SLOGroup.SLOs)) for _, slo := range r.SLOGroup.SLOs { // Add extra labels. - slo.Labels = mergeLabels(slo.Labels, r.ExtraLabels) + slo.Labels = mergeLabels(slo.Labels, r.ExtraLabels, r.ExtraFilterLabels) // Generate SLO result. - result, err := s.generateSLO(ctx, r.Info, slo) + result, err := s.generateSLO(ctx, r.Info, slo, r.ExtraFilterLabels) if err != nil { return nil, fmt.Errorf("could not generate %q slo: %w", slo.ID, err) } @@ -136,7 +139,7 @@ func (s Service) Generate(ctx context.Context, r Request) (*Response, error) { }, nil } -func (s Service) generateSLO(ctx context.Context, info info.Info, slo prometheus.SLO) (*SLOResult, error) { +func (s Service) generateSLO(ctx context.Context, info info.Info, slo prometheus.SLO, extraFilterLabels map[string]string) (*SLOResult, error) { logger := s.logger.WithCtxValues(ctx).WithValues(log.Kv{"slo": slo.ID}) // Generate the MWMB alerts. @@ -152,14 +155,14 @@ func (s Service) generateSLO(ctx context.Context, info info.Info, slo prometheus logger.Infof("Multiwindow-multiburn alerts generated") // Generate SLI recording rules. - sliRecordingRules, err := s.sliRecordRuleGen.GenerateSLIRecordingRules(ctx, slo, *as) + sliRecordingRules, err := s.sliRecordRuleGen.GenerateSLIRecordingRules(ctx, slo, *as, extraFilterLabels) if err != nil { return nil, fmt.Errorf("could not generate Prometheus sli recording rules: %w", err) } logger.WithValues(log.Kv{"rules": len(sliRecordingRules)}).Infof("SLI recording rules generated") // Generate Metadata recording rules. - metaRecordingRules, err := s.metaRecordRuleGen.GenerateMetadataRecordingRules(ctx, info, slo, *as) + metaRecordingRules, err := s.metaRecordRuleGen.GenerateMetadataRecordingRules(ctx, info, slo, *as, extraFilterLabels) if err != nil { return nil, fmt.Errorf("could not generate Prometheus metadata recording rules: %w", err) } diff --git a/internal/app/kubecontroller/handler.go b/internal/app/kubecontroller/handler.go index eeb77151..bc65e61c 100644 --- a/internal/app/kubecontroller/handler.go +++ b/internal/app/kubecontroller/handler.go @@ -37,11 +37,12 @@ type KubeStatusStorer interface { // HandlerConfig is the controller handler configuration. type HandlerConfig struct { - Generator Generator - SpecLoader SpecLoader - Repository Repository - KubeStatusStorer KubeStatusStorer - ExtraLabels map[string]string + Generator Generator + SpecLoader SpecLoader + Repository Repository + KubeStatusStorer KubeStatusStorer + ExtraLabels map[string]string + ExtraFilterLabels map[string]string // IgnoreHandleBefore makes the handles of objects with a success state and no spec change, // be ignored if the last success is less than this setting. // Be aware that this setting should be less than the controller resync interval. @@ -66,6 +67,10 @@ func (c *HandlerConfig) defaults() error { c.ExtraLabels = map[string]string{} } + if c.ExtraFilterLabels == nil { + c.ExtraFilterLabels = map[string]string{} + } + if c.Repository == nil { return fmt.Errorf("repository is required") } @@ -88,6 +93,7 @@ type handler struct { repository Repository kubeStatusStorer KubeStatusStorer extraLabels map[string]string + extraFilterLabels map[string]string ignoreHandleBefore time.Duration logger log.Logger } @@ -103,6 +109,7 @@ func NewHandler(config HandlerConfig) (controller.Handler, error) { repository: config.Repository, kubeStatusStorer: config.KubeStatusStorer, extraLabels: config.ExtraLabels, + extraFilterLabels: config.ExtraFilterLabels, ignoreHandleBefore: config.IgnoreHandleBefore, logger: config.Logger, }, nil @@ -151,8 +158,9 @@ func (h handler) handlePrometheusServiceLevelV1(ctx context.Context, psl *slothv Mode: info.ModeControllerGenKubernetes, Spec: fmt.Sprintf("%s/%s", slothv1.SchemeGroupVersion.Group, slothv1.SchemeGroupVersion.Version), }, - ExtraLabels: h.extraLabels, - SLOGroup: model.SLOGroup, + ExtraLabels: h.extraLabels, + ExtraFilterLabels: h.extraFilterLabels, + SLOGroup: model.SLOGroup, } resp, err := h.generator.Generate(ctx, req) if err != nil { diff --git a/internal/prometheus/recording_rules.go b/internal/prometheus/recording_rules.go index b9f4a6c4..57866155 100644 --- a/internal/prometheus/recording_rules.go +++ b/internal/prometheus/recording_rules.go @@ -15,7 +15,7 @@ import ( ) // sliRulesgenFunc knows how to generate an SLI recording rule for a specific time window. -type sliRulesgenFunc func(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup) (*rulefmt.Rule, error) +type sliRulesgenFunc func(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) (*rulefmt.Rule, error) type sliRecordingRulesGenerator struct { genFunc sliRulesgenFunc @@ -31,16 +31,16 @@ var OptimizedSLIRecordingRulesGenerator = sliRecordingRulesGenerator{genFunc: op // Normally these rules are used by the SLO alerts. var SLIRecordingRulesGenerator = sliRecordingRulesGenerator{genFunc: factorySLIRecordGenerator} -func optimizedFactorySLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup) (*rulefmt.Rule, error) { +func optimizedFactorySLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) (*rulefmt.Rule, error) { // Optimize the rules that are for the total period time window. if window == slo.TimeWindow { - return optimizedSLIRecordGenerator(slo, window, alerts.PageQuick.ShortWindow) + return optimizedSLIRecordGenerator(slo, window, alerts.PageQuick.ShortWindow, extraFilterLabels) } - return factorySLIRecordGenerator(slo, window, alerts) + return factorySLIRecordGenerator(slo, window, alerts, extraFilterLabels) } -func (s sliRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Context, slo SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) { +func (s sliRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Context, slo SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) { // Get the windows we need the recording rules. windows := getAlertGroupWindows(alerts) windows = append(windows, slo.TimeWindow) // Add the total time window as a handy helper. @@ -48,7 +48,7 @@ func (s sliRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Contex // Generate the rules rules := make([]rulefmt.Rule, 0, len(windows)) for _, window := range windows { - rule, err := s.genFunc(slo, window, alerts) + rule, err := s.genFunc(slo, window, alerts, extraFilterLabels) if err != nil { return nil, fmt.Errorf("could not create %q SLO rule for window %s: %w", slo.ID, window, err) } @@ -62,20 +62,20 @@ const ( tplKeyWindow = "window" ) -func factorySLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup) (*rulefmt.Rule, error) { +func factorySLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) (*rulefmt.Rule, error) { switch { // Event based SLI. case slo.SLI.Events != nil: - return eventsSLIRecordGenerator(slo, window, alerts) + return eventsSLIRecordGenerator(slo, window, alerts, extraFilterLabels) // Raw based SLI. case slo.SLI.Raw != nil: - return rawSLIRecordGenerator(slo, window, alerts) + return rawSLIRecordGenerator(slo, window, alerts, extraFilterLabels) } return nil, fmt.Errorf("invalid SLI type") } -func rawSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup) (*rulefmt.Rule, error) { +func rawSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) (*rulefmt.Rule, error) { // Render with our templated data. sliExprTpl := fmt.Sprintf(`(%s)`, slo.SLI.Raw.ErrorRatioQuery) tpl, err := template.New("sliExpr").Option("missingkey=error").Parse(sliExprTpl) @@ -85,9 +85,11 @@ func rawSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlert strWindow := timeDurationToPromStr(window) var b bytes.Buffer - err = tpl.Execute(&b, map[string]string{ - tplKeyWindow: strWindow, - }) + err = tpl.Execute(&b, mergeLabels(extraFilterLabels, + map[string]string{ + tplKeyWindow: strWindow, + }), + ) if err != nil { return nil, fmt.Errorf("could not render SLI expression template: %w", err) } @@ -105,7 +107,7 @@ func rawSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlert }, nil } -func eventsSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup) (*rulefmt.Rule, error) { +func eventsSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) (*rulefmt.Rule, error) { const sliExprTplFmt = `(%s) / (%s) @@ -121,9 +123,11 @@ func eventsSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAl strWindow := timeDurationToPromStr(window) var b bytes.Buffer - err = tpl.Execute(&b, map[string]string{ - tplKeyWindow: strWindow, - }) + err = tpl.Execute(&b, mergeLabels(extraFilterLabels, + map[string]string{ + tplKeyWindow: strWindow, + }), + ) if err != nil { return nil, fmt.Errorf("could not render SLI expression template: %w", err) } @@ -147,7 +151,7 @@ func eventsSLIRecordGenerator(slo SLO, window time.Duration, alerts alert.MWMBAl // // The way this optimization is made is using one SLI recording rule (the one with the shortest window to // reduce the downsampling, e.g 5m) and make an average over time on that rule for the window time range. -func optimizedSLIRecordGenerator(slo SLO, window, shortWindow time.Duration) (*rulefmt.Rule, error) { +func optimizedSLIRecordGenerator(slo SLO, window, shortWindow time.Duration, extraFilterLabels map[string]string) (*rulefmt.Rule, error) { // Averages over ratios (average over average) is statistically incorrect, so we do // aggregate all ratios on the time window and then divide with the aggregation of all the full ratios // that is 1 (thats why we can use `count`), giving use a correct ratio of ratios: @@ -163,7 +167,10 @@ count_over_time({{.metric}}{{.filter}}[{{.window}}]) } shortWindowSLIRec := slo.GetSLIErrorMetric(shortWindow) - filter := labelsToPromFilter(slo.GetSLOIDPromLabels()) + filter := labelsToPromFilter(mergeLabels( + slo.GetSLOIDPromLabels(), + extraFilterLabels, + )) // Render with our templated data. tpl, err := template.New("sliExpr").Option("missingkey=error").Parse(sliExprTplFmt) @@ -202,7 +209,7 @@ type metadataRecordingRulesGenerator bool // from an SLO. const MetadataRecordingRulesGenerator = metadataRecordingRulesGenerator(false) -func (m metadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) { +func (m metadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) { labels := mergeLabels(slo.GetSLOIDPromLabels(), slo.Labels) // Metatada Recordings. @@ -218,7 +225,10 @@ func (m metadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx cont sloObjectiveRatio := slo.Objective / 100 - sloFilter := labelsToPromFilter(slo.GetSLOIDPromLabels()) + sloFilter := labelsToPromFilter(mergeLabels( + slo.GetSLOIDPromLabels(), + extraFilterLabels, + )) var currentBurnRateExpr bytes.Buffer err := burnRateRecordingExprTpl.Execute(¤tBurnRateExpr, map[string]string{ diff --git a/internal/prometheus/recording_rules_test.go b/internal/prometheus/recording_rules_test.go index 6e518d78..8851b905 100644 --- a/internal/prometheus/recording_rules_test.go +++ b/internal/prometheus/recording_rules_test.go @@ -36,7 +36,7 @@ func getAlertGroup() alert.MWMBAlertGroup { func TestGenerateSLIRecordingRules(t *testing.T) { type generator interface { - GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) + GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) } tests := map[string]struct { @@ -493,7 +493,7 @@ func TestGenerateSLIRecordingRules(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - gotRules, err := test.generator().GenerateSLIRecordingRules(context.TODO(), test.slo, test.alertGroup) + gotRules, err := test.generator().GenerateSLIRecordingRules(context.TODO(), test.slo, test.alertGroup, map[string]string{}) if test.expErr { assert.Error(err) @@ -618,7 +618,7 @@ slo:error_budget:ratio{sloth_id="test", sloth_service="test-svc", sloth_slo="tes t.Run(name, func(t *testing.T) { assert := assert.New(t) - gotRules, err := prometheus.MetadataRecordingRulesGenerator.GenerateMetadataRecordingRules(context.TODO(), test.info, test.slo, test.alertGroup) + gotRules, err := prometheus.MetadataRecordingRulesGenerator.GenerateMetadataRecordingRules(context.TODO(), test.info, test.slo, test.alertGroup, map[string]string{}) if test.expErr { assert.Error(err)