slok · Goorsky1 · Aug 3, 2023
@@ -36,21 +36,26 @@ type generateCommand struct {
 	disableAlerts         bool
 	disableOptimizedRules bool
 	extraLabels           map[string]string
+	extraFilterLabels     map[string]string
 	sliPluginsPaths       []string
 	sloPeriodWindowsPath  string
 	sloPeriod             string
 }
 
 // NewGenerateCommand returns the generate command.
 func NewGenerateCommand(app *kingpin.Application) Command {
-	c := &generateCommand{extraLabels: map[string]string{}}
+	c := &generateCommand{
+		extraLabels:       map[string]string{},
+		extraFilterLabels: map[string]string{},
+	}
 	cmd := app.Command("generate", "Generates Prometheus SLOs.")
 	cmd.Flag("input", "SLO spec input file path or directory (if directory is used, slos will be discovered recursively and out must be a directory).").Short('i').StringVar(&c.slosInput)
 	cmd.Flag("out", "Generated rules output file path or directory. If `-` it will use stdout (if input is a directory this must be a directory).").Default("-").Short('o').StringVar(&c.slosOut)
 	cmd.Flag("fs-exclude", "Filter regex to ignore matched discovered SLO file paths (used with directory based input/output).").Short('e').StringVar(&c.slosExcludeRegex)
 	cmd.Flag("fs-include", "Filter regex to include matched discovered SLO file paths, everything else will be ignored. Exclude has preference (used with directory based input/output).").Short('n').StringVar(&c.slosIncludeRegex)
 
 	cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels)
+	cmd.Flag("extra-filter-labels", "Extra labels that will be added to all the generated Prometheus rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. ('key=value' form, can be repeated).").StringMapVar(&c.extraFilterLabels)
 	cmd.Flag("disable-recordings", "Disables recording rules generation.").BoolVar(&c.disableRecordings)
 	cmd.Flag("disable-alerts", "Disables alert rules generation.").BoolVar(&c.disableAlerts)
 	cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths)
@@ -245,6 +250,7 @@ func (g generateCommand) Run(ctx context.Context, config RootConfig) error {
 		disableAlerts:         g.disableAlerts,
 		disableOptimizedRules: g.disableOptimizedRules,
 		extraLabels:           g.extraLabels,
+		extraFilterLabels:     g.extraFilterLabels,
 	}
 
 	for _, genTarget := range genTargets {
@@ -305,6 +311,7 @@ type generator struct {
 	disableAlerts         bool
 	disableOptimizedRules bool
 	extraLabels           map[string]string
+	extraFilterLabels     map[string]string
 }
 
 // GeneratePrometheus generates the SLOs based on a raw regular Prometheus spec format input and outs a Prometheus raw yaml.
@@ -433,9 +440,10 @@ func (g generator) generateRules(ctx context.Context, info info.Info, slos prome
 	}
 
 	result, err := controller.Generate(ctx, generate.Request{
-		ExtraLabels: g.extraLabels,
-		Info:        info,
-		SLOGroup:    slos,
+		ExtraLabels:       g.extraLabels,
+		ExtraFilterLabels: g.extraFilterLabels,
+		Info:              info,
+		SLOGroup:          slos,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("could not generate prometheus rules: %w", err)

@@ -53,6 +53,7 @@ const (
 
 type kubeControllerCommand struct {
 	extraLabels           map[string]string
+	extraFilterLabels     map[string]string
 	workers               int
 	kubeConfig            string
 	kubeContext           string
@@ -73,7 +74,10 @@ type kubeControllerCommand struct {
 
 // NewKubeControllerCommand returns the Kubernetes controller command.
 func NewKubeControllerCommand(app *kingpin.Application) Command {
-	c := &kubeControllerCommand{extraLabels: map[string]string{}}
+	c := &kubeControllerCommand{
+		extraLabels:       map[string]string{},
+		extraFilterLabels: map[string]string{},
+	}
 	cmd := app.Command("kubernetes-controller", "Runs Sloth in Kubernetes controller/operator mode.")
 	cmd.Alias("controller")
 	cmd.Alias("k8s-controller")
@@ -92,6 +96,7 @@ func NewKubeControllerCommand(app *kingpin.Application) Command {
 	cmd.Flag("hot-reload-addr", "The listen address for hot-reloading components that allow it.").Default(":8082").StringVar(&c.hotReloadAddr)
 	cmd.Flag("hot-reload-path", "The webhook path for hot-reloading components that allow it.").Default("/-/reload").StringVar(&c.hotReloadPath)
 	cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels)
+	cmd.Flag("extra-filter-labels", "Extra labels that will be added to all the generated Prometheus rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. ('key=value' form, can be repeated).").StringMapVar(&c.extraFilterLabels)
 	cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths)
 	cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath)
 	cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod)
@@ -318,12 +323,13 @@ func (k kubeControllerCommand) Run(ctx context.Context, config RootConfig) error
 
 		// Create handler.
 		config := kubecontroller.HandlerConfig{
-			Generator:        generator,
-			SpecLoader:       k8sprometheus.NewCRSpecLoader(pluginRepo, sloPeriod),
-			Repository:       k8sprometheus.NewPrometheusOperatorCRDRepo(ksvc, logger),
-			KubeStatusStorer: ksvc,
-			ExtraLabels:      k.extraLabels,
-			Logger:           logger,
+			Generator:         generator,
+			SpecLoader:        k8sprometheus.NewCRSpecLoader(pluginRepo, sloPeriod),
+			Repository:        k8sprometheus.NewPrometheusOperatorCRDRepo(ksvc, logger),
+			KubeStatusStorer:  ksvc,
+			ExtraLabels:       k.extraLabels,
+			ExtraFilterLabels: k.extraFilterLabels,
+			Logger:            logger,
 		}
 		handler, err := kubecontroller.NewHandler(config)
 		if err != nil {

@@ -24,19 +24,24 @@ type validateCommand struct {
 	slosExcludeRegex     string
 	slosIncludeRegex     string
 	extraLabels          map[string]string
+	extraFilterLabels    map[string]string
 	sliPluginsPaths      []string
 	sloPeriodWindowsPath string
 	sloPeriod            string
 }
 
 // NewValidateCommand returns the validate command.
 func NewValidateCommand(app *kingpin.Application) Command {
-	c := &validateCommand{extraLabels: map[string]string{}}
+	c := &validateCommand{
+		extraLabels:       map[string]string{},
+		extraFilterLabels: map[string]string{},
+	}
 	cmd := app.Command("validate", "Validates the SLO manifests and generation of Prometheus SLOs.")
 	cmd.Flag("input", "SLO spec discovery path, will discover recursively all YAML files.").Short('i').Required().StringVar(&c.slosInput)
 	cmd.Flag("fs-exclude", "Filter regex to ignore matched discovered SLO file paths.").Short('e').StringVar(&c.slosExcludeRegex)
 	cmd.Flag("fs-include", "Filter regex to include matched discovered SLO file paths, everything else will be ignored. Exclude has preference.").Short('n').StringVar(&c.slosIncludeRegex)
 	cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels)
+	cmd.Flag("extra-filter-labels", "Extra labels that will be added to all the generated Prometheus rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user. ('key=value' form, can be repeated).").StringMapVar(&c.extraFilterLabels)
 	cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths)
 	cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath)
 	cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod)
@@ -126,9 +131,10 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error {
 		splittedSLOsData := splitYAML(slxData)
 
 		gen := generator{
-			logger:      log.Noop,
-			windowsRepo: windowsRepo,
-			extraLabels: v.extraLabels,
+			logger:            log.Noop,
+			windowsRepo:       windowsRepo,
+			extraLabels:       v.extraLabels,
+			extraFilterLabels: v.extraFilterLabels,
 		}
 
 		// Prepare file validation result and start validation result for every SLO in the file.

@@ -47,6 +47,9 @@ spec:
             {{- range $key, $val := .Values.sloth.extraLabels }}
             - --extra-labels={{ $key }}={{ $val }}
             {{- end}}
+            {{- range $key, $val := .Values.sloth.extraFilterLabels }}
+            - --extra-filter-labels={{ $key }}={{ $val }}
+            {{- end}}
             {{- if .Values.commonPlugins.enabled }}
             - --sli-plugins-path=/plugins
             {{- end }}

@@ -27,6 +27,10 @@ func customValues() msi {
 				"k1": "v1",
 				"k2": "v2",
 			},
+			"extraFilterLabels": msi{
+				"k3": "v3",
+				"k4": "v4",
+			},
 		},
 
 		"commonPlugins": msi{

@@ -23,6 +23,7 @@ sloth:
   labelSelector: ""     # Sloth will handle only the ones that match the selector.
   namespace: ""         # The namespace where sloth will the CRs to process.
   extraLabels: {}       # Labels that will be added to all the generated SLO Rules.
+  extraFilterLabels: {} # Labels that will be added to all the generated SLO Rules, but also will be added as filters to the generated recording rules. Can be used as templates in queries provided by the user.
   defaultSloPeriod: ""  # The slo period used by sloth (e.g. 30d).
   optimizedRules: true  # Reduce prom load for calculating period window burnrates.
   debug:

@@ -13,15 +13,15 @@ type noopSLIRecordingRulesGenerator bool
 
 const NoopSLIRecordingRulesGenerator = noopSLIRecordingRulesGenerator(false)
 
-func (noopSLIRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) {
+func (noopSLIRecordingRulesGenerator) GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) {
 	return nil, nil
 }
 
 type noopMetadataRecordingRulesGenerator bool
 
 const NoopMetadataRecordingRulesGenerator = noopMetadataRecordingRulesGenerator(false)
 
-func (noopMetadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error) {
+func (noopMetadataRecordingRulesGenerator) GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error) {
 	return nil, nil
 }
 

@@ -53,12 +53,12 @@ type AlertGenerator interface {
 
 // SLIRecordingRulesGenerator knows how to generate SLI recording rules.
 type SLIRecordingRulesGenerator interface {
-	GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error)
+	GenerateSLIRecordingRules(ctx context.Context, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error)
 }
 
 // MetadataRecordingRulesGenerator knows how to generate metadata recording rules.
 type MetadataRecordingRulesGenerator interface {
-	GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup) ([]rulefmt.Rule, error)
+	GenerateMetadataRecordingRules(ctx context.Context, info info.Info, slo prometheus.SLO, alerts alert.MWMBAlertGroup, extraFilterLabels map[string]string) ([]rulefmt.Rule, error)
 }
 
 // SLOAlertRulesGenerator knows hot to generate SLO alert rules.
@@ -96,6 +96,9 @@ type Request struct {
 	Info info.Info
 	// ExtraLabels are the extra labels added to the SLOs on execution time.
 	ExtraLabels map[string]string
+	// ExtraFilterLabels are the extra labels added to the SLOs on execution time and also will be added as filters to the generated recording rules.
+	// Can be used as templates in queries provided by the user.
+	ExtraFilterLabels map[string]string
 	// SLOGroup are the SLOs group that will be used to generate the SLO results and Prom rules.
 	SLOGroup prometheus.SLOGroup
 }
@@ -120,10 +123,10 @@ func (s Service) Generate(ctx context.Context, r Request) (*Response, error) {
 	results := make([]SLOResult, 0, len(r.SLOGroup.SLOs))
 	for _, slo := range r.SLOGroup.SLOs {
 		// Add extra labels.
-		slo.Labels = mergeLabels(slo.Labels, r.ExtraLabels)
+		slo.Labels = mergeLabels(slo.Labels, r.ExtraLabels, r.ExtraFilterLabels)
 
 		// Generate SLO result.
-		result, err := s.generateSLO(ctx, r.Info, slo)
+		result, err := s.generateSLO(ctx, r.Info, slo, r.ExtraFilterLabels)
 		if err != nil {
 			return nil, fmt.Errorf("could not generate %q slo: %w", slo.ID, err)
 		}
@@ -136,7 +139,7 @@ func (s Service) Generate(ctx context.Context, r Request) (*Response, error) {
 	}, nil
 }
 
-func (s Service) generateSLO(ctx context.Context, info info.Info, slo prometheus.SLO) (*SLOResult, error) {
+func (s Service) generateSLO(ctx context.Context, info info.Info, slo prometheus.SLO, extraFilterLabels map[string]string) (*SLOResult, error) {
 	logger := s.logger.WithCtxValues(ctx).WithValues(log.Kv{"slo": slo.ID})
 
 	// Generate the MWMB alerts.
@@ -152,14 +155,14 @@ func (s Service) generateSLO(ctx context.Context, info info.Info, slo prometheus
 	logger.Infof("Multiwindow-multiburn alerts generated")
 
 	// Generate SLI recording rules.
-	sliRecordingRules, err := s.sliRecordRuleGen.GenerateSLIRecordingRules(ctx, slo, *as)
+	sliRecordingRules, err := s.sliRecordRuleGen.GenerateSLIRecordingRules(ctx, slo, *as, extraFilterLabels)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate Prometheus sli recording rules: %w", err)
 	}
 	logger.WithValues(log.Kv{"rules": len(sliRecordingRules)}).Infof("SLI recording rules generated")
 
 	// Generate Metadata recording rules.
-	metaRecordingRules, err := s.metaRecordRuleGen.GenerateMetadataRecordingRules(ctx, info, slo, *as)
+	metaRecordingRules, err := s.metaRecordRuleGen.GenerateMetadataRecordingRules(ctx, info, slo, *as, extraFilterLabels)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate Prometheus metadata recording rules: %w", err)
 	}

@@ -37,11 +37,12 @@ type KubeStatusStorer interface {
 
 // HandlerConfig is the controller handler configuration.
 type HandlerConfig struct {
-	Generator        Generator
-	SpecLoader       SpecLoader
-	Repository       Repository
-	KubeStatusStorer KubeStatusStorer
-	ExtraLabels      map[string]string
+	Generator         Generator
+	SpecLoader        SpecLoader
+	Repository        Repository
+	KubeStatusStorer  KubeStatusStorer
+	ExtraLabels       map[string]string
+	ExtraFilterLabels map[string]string
 	// IgnoreHandleBefore makes the handles of objects with a success state and no spec change,
 	// be ignored if the last success is less than this setting.
 	// Be aware that this setting should be less than the controller resync interval.
@@ -66,6 +67,10 @@ func (c *HandlerConfig) defaults() error {
 		c.ExtraLabels = map[string]string{}
 	}
 
+	if c.ExtraFilterLabels == nil {
+		c.ExtraFilterLabels = map[string]string{}
+	}
+
 	if c.Repository == nil {
 		return fmt.Errorf("repository is required")
 	}
@@ -88,6 +93,7 @@ type handler struct {
 	repository         Repository
 	kubeStatusStorer   KubeStatusStorer
 	extraLabels        map[string]string
+	extraFilterLabels  map[string]string
 	ignoreHandleBefore time.Duration
 	logger             log.Logger
 }
@@ -103,6 +109,7 @@ func NewHandler(config HandlerConfig) (controller.Handler, error) {
 		repository:         config.Repository,
 		kubeStatusStorer:   config.KubeStatusStorer,
 		extraLabels:        config.ExtraLabels,
+		extraFilterLabels:  config.ExtraFilterLabels,
 		ignoreHandleBefore: config.IgnoreHandleBefore,
 		logger:             config.Logger,
 	}, nil
@@ -151,8 +158,9 @@ func (h handler) handlePrometheusServiceLevelV1(ctx context.Context, psl *slothv
 			Mode:    info.ModeControllerGenKubernetes,
 			Spec:    fmt.Sprintf("%s/%s", slothv1.SchemeGroupVersion.Group, slothv1.SchemeGroupVersion.Version),
 		},
-		ExtraLabels: h.extraLabels,
-		SLOGroup:    model.SLOGroup,
+		ExtraLabels:       h.extraLabels,
+		ExtraFilterLabels: h.extraFilterLabels,
+		SLOGroup:          model.SLOGroup,
 	}
 	resp, err := h.generator.Generate(ctx, req)
 	if err != nil {