+(Appears on:PerTenantQueryLimitSpec) +
+BlockedQuerySpec defines the rule spec for queries to be blocked.
+| Field | +Description | +
|---|---|
+hash+ +int32 + + |
+
+(Optional)
+ Hash is a 32-bit FNV-1 hash of the query string. + |
+
+pattern+ +string + + |
+
+(Optional)
+ Pattern defines the pattern matching the queries to be blocked. + |
+
+regex+ +bool + + |
+
+(Optional)
+ Regex defines if the pattern is a regular expression. If false the pattern will be used only for exact matches. + |
+
+types+ + +BlockedQueryTypes + + + |
+
+(Optional)
+ Types defines the list of query types that should be considered for blocking. + |
+
string alias)
+BlockedQueryType defines which type of query a blocked query should apply to.
+| Value | +Description | +
|---|---|
"filter" |
+BlockedQueryFilter is used, when the blocked query should apply to queries using a log filter. + |
+
"limited" |
+BlockedQueryLimited is used, when the blocked query should apply to queries without a filter or a metric aggregation. + |
+
"metric" |
+BlockedQueryMetric is used, when the blocked query should apply to queries with an aggregation. + |
+
[]github.com/grafana/loki/operator/apis/loki/v1.BlockedQueryType alias)
++(Appears on:BlockedQuerySpec) +
+BlockedQueryTypes defines a slice of BlockedQueryType values to be used for a blocked query.
+(Appears on:MTLSSpec, OIDCSpec, ObjectStorageTLSSpec) @@ -1067,7 +1168,7 @@ MemberListSpec ## IngestionLimitSpec { #loki-grafana-com-v1-IngestionLimitSpec }
-(Appears on:LimitsTemplateSpec) +(Appears on:LimitsTemplateSpec, PerTenantLimitsTemplateSpec)
IngestionLimitSpec defines the limits applied at the ingestion path.
@@ -1268,8 +1369,8 @@ LimitsTemplateSpectenants+(Appears on:LimitsSpec) +
+LimitsTemplateSpec defines the limits applied at ingestion or query path.
+| Field | +Description | +
|---|---|
+ingestion+ + +IngestionLimitSpec + + + |
+
+(Optional)
+ IngestionLimits defines the limits applied on ingested log streams. + |
+
+queries+ + +PerTenantQueryLimitSpec + + + |
+
+(Optional)
+ QueryLimits defines the limit applied on querying log streams. + |
+
+retention+ + +RetentionLimitSpec + + + |
+
+(Optional)
+ Retention defines how long logs are kept in storage. + |
+
+(Appears on:PerTenantLimitsTemplateSpec) +
+PerTenantQueryLimitSpec defines the limits applied to per tenant query path.
+| Field | +Description | +
|---|---|
+QueryLimitSpec+ + +QueryLimitSpec + + + |
++ | +
+blocked+ + +[]BlockedQuerySpec + + + |
+
+(Optional)
+ Blocked defines the list of rules to block matching queries. + |
+
string alias)
@@ -2762,7 +2967,7 @@ Setting this to an empty array disables admin groups.
## QueryLimitSpec { #loki-grafana-com-v1-QueryLimitSpec }-(Appears on:LimitsTemplateSpec) +(Appears on:LimitsTemplateSpec, PerTenantQueryLimitSpec)
QueryLimitSpec defines the limits applies at the query path.
@@ -3588,7 +3793,7 @@ IMPORTANT: Make sure that the replication factor defined is less than or equal t ## RetentionLimitSpec { #loki-grafana-com-v1-RetentionLimitSpec }-(Appears on:LimitsTemplateSpec) +(Appears on:LimitsTemplateSpec, PerTenantLimitsTemplateSpec)
RetentionLimitSpec controls how long logs will be kept in storage.
diff --git a/operator/internal/manifests/config_test.go b/operator/internal/manifests/config_test.go index 5df65eb709cc5..b58cac99e1083 100644 --- a/operator/internal/manifests/config_test.go +++ b/operator/internal/manifests/config_test.go @@ -84,7 +84,7 @@ func randomConfigOptions() Options { MaxQuerySeries: rand.Int31(), }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ uuid.New().String(): { IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: rand.Int31(), @@ -97,10 +97,12 @@ func randomConfigOptions() Options { PerStreamRateLimit: rand.Int31(), PerStreamRateLimitBurst: rand.Int31(), }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxEntriesLimitPerQuery: rand.Int31(), - MaxChunksPerQuery: rand.Int31(), - MaxQuerySeries: rand.Int31(), + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxEntriesLimitPerQuery: rand.Int31(), + MaxChunksPerQuery: rand.Int31(), + MaxQuerySeries: rand.Int31(), + }, }, }, }, @@ -375,7 +377,7 @@ func TestConfigOptions_RetentionConfig(t *testing.T) { Days: 14, }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "development": { Retention: &lokiv1.RetentionLimitSpec{ Days: 3, @@ -394,7 +396,7 @@ func TestConfigOptions_RetentionConfig(t *testing.T) { spec: lokiv1.LokiStackSpec{ Size: lokiv1.SizeOneXExtraSmall, Limits: &lokiv1.LimitsSpec{ - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "development": { Retention: &lokiv1.RetentionLimitSpec{ Days: 3, @@ -1092,10 +1094,12 @@ func TestConfigOptions_RulerOverrides_OCPUserWorkloadOnlyEnabled(t *testing.T) { Enabled: true, }, Limits: &lokiv1.LimitsSpec{ - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "application": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "5m", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "5m", + }, }, }, }, @@ -1132,9 +1136,11 @@ func TestConfigOptions_RulerOverrides_OCPUserWorkloadOnlyEnabled(t *testing.T) { }, wantOverridesOptions: map[string]config.LokiOverrides{ "application": { - Limits: lokiv1.LimitsTemplateSpec{ - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "5m", + Limits: lokiv1.PerTenantLimitsTemplateSpec{ + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "5m", + }, }, }, Ruler: config.RulerOverrides{ diff --git a/operator/internal/manifests/internal/config/build.go b/operator/internal/manifests/internal/config/build.go index 399e9efd493c8..f599b791a7949 100644 --- a/operator/internal/manifests/internal/config/build.go +++ b/operator/internal/manifests/internal/config/build.go @@ -4,6 +4,8 @@ import ( "bytes" "embed" "io" + "reflect" + "strings" "text/template" "github.com/ViaQ/logerr/v2/kverrors" @@ -27,7 +29,9 @@ var ( lokiConfigYAMLTmpl = template.Must(template.ParseFS(lokiConfigYAMLTmplFile, "loki-config.yaml")) - lokiRuntimeConfigYAMLTmpl = template.Must(template.ParseFS(lokiRuntimeConfigYAMLTmplFile, "loki-runtime-config.yaml")) + lokiRuntimeConfigYAMLTmpl = template.Must(template.New("loki-runtime-config.yaml").Funcs(template.FuncMap{ + "yamlBlock": yamlBlock, + }).ParseFS(lokiRuntimeConfigYAMLTmplFile, "loki-runtime-config.yaml")) ) // Build builds a loki stack configuration files @@ -54,3 +58,9 @@ func Build(opts Options) ([]byte, []byte, error) { } return cfg, rcfg, nil } + +func yamlBlock(indent string, in reflect.Value) string { + inStr := in.String() + lines := strings.Split(strings.TrimRight(inStr, "\n"), "\n") + return strings.Join(lines, "\n"+indent) +} diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index fd254b7ee1e9d..8b448008c82d8 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -444,6 +444,15 @@ overrides: ingestion_burst_size_mb: 5 max_global_streams_per_user: 1 max_chunks_per_query: 1000000 + blocked_queries: + - hash: 12345 + types: metric,limited + - pattern: | + .*prod.* + regex: true + - types: metric + - pattern: | + sum(rate({env="prod"}[1m])) ` opts := Options{ Stack: lokiv1.LokiStackSpec{ @@ -472,15 +481,33 @@ overrides: CardinalityLimit: 100000, }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "test-a": { IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, IngestionBurstSize: 5, MaxGlobalStreamsPerTenant: 1, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, + Blocked: []lokiv1.BlockedQuerySpec{ + { + Hash: 12345, + Types: lokiv1.BlockedQueryTypes{lokiv1.BlockedQueryMetric, lokiv1.BlockedQueryLimited}, + }, + { + Pattern: ".*prod.*", + Regex: true, + }, + { + Types: lokiv1.BlockedQueryTypes{lokiv1.BlockedQueryMetric}, + }, + { + Pattern: `sum(rate({env="prod"}[1m]))`, + }, + }, }, }, }, @@ -488,14 +515,32 @@ overrides: }, Overrides: map[string]LokiOverrides{ "test-a": { - Limits: lokiv1.LimitsTemplateSpec{ + Limits: lokiv1.PerTenantLimitsTemplateSpec{ IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, MaxGlobalStreamsPerTenant: 1, IngestionBurstSize: 5, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, + Blocked: []lokiv1.BlockedQuerySpec{ + { + Hash: 12345, + Types: lokiv1.BlockedQueryTypes{lokiv1.BlockedQueryMetric, lokiv1.BlockedQueryLimited}, + }, + { + Pattern: ".*prod.*", + Regex: true, + }, + { + Types: lokiv1.BlockedQueryTypes{lokiv1.BlockedQueryMetric}, + }, + { + Pattern: `sum(rate({env="prod"}[1m]))`, + }, + }, }, }, }, @@ -1977,15 +2022,17 @@ overrides: }, }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "test-a": { IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, IngestionBurstSize: 5, MaxGlobalStreamsPerTenant: 1, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, }, Retention: &lokiv1.RetentionLimitSpec{ Days: 7, @@ -2003,14 +2050,16 @@ overrides: }, Overrides: map[string]LokiOverrides{ "test-a": { - Limits: lokiv1.LimitsTemplateSpec{ + Limits: lokiv1.PerTenantLimitsTemplateSpec{ IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, IngestionBurstSize: 5, MaxGlobalStreamsPerTenant: 1, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, }, Retention: &lokiv1.RetentionLimitSpec{ Days: 7, @@ -4350,15 +4399,17 @@ overrides: CardinalityLimit: 100000, }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "test-a": { IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, IngestionBurstSize: 5, MaxGlobalStreamsPerTenant: 1, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, }, }, }, @@ -4366,14 +4417,16 @@ overrides: }, Overrides: map[string]LokiOverrides{ "test-a": { - Limits: lokiv1.LimitsTemplateSpec{ + Limits: lokiv1.PerTenantLimitsTemplateSpec{ IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, MaxGlobalStreamsPerTenant: 1, IngestionBurstSize: 5, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, }, }, }, @@ -4646,15 +4699,17 @@ overrides: CardinalityLimit: 100000, }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "test-a": { IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, IngestionBurstSize: 5, MaxGlobalStreamsPerTenant: 1, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, }, }, }, @@ -4662,14 +4717,16 @@ overrides: }, Overrides: map[string]LokiOverrides{ "test-a": { - Limits: lokiv1.LimitsTemplateSpec{ + Limits: lokiv1.PerTenantLimitsTemplateSpec{ IngestionLimits: &lokiv1.IngestionLimitSpec{ IngestionRate: 2, MaxGlobalStreamsPerTenant: 1, IngestionBurstSize: 5, }, - QueryLimits: &lokiv1.QueryLimitSpec{ - MaxChunksPerQuery: 1000000, + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + MaxChunksPerQuery: 1000000, + }, }, }, }, diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml index 8b2fe60b23f8e..421426cb3911f 100644 --- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml @@ -54,6 +54,24 @@ overrides: {{- if $spec.QueryLimits.CardinalityLimit }} cardinality_limit: {{ $spec.QueryLimits.CardinalityLimit }} {{- end }} + {{- with $l.Blocked }} + blocked_queries: + {{- range $blockedQuery := . }} + - {{ with $blockedQuery.Pattern -}} + pattern: | + {{ . | yamlBlock " " }} + {{ end -}} + {{- with $blockedQuery.Regex }} + regex: {{ . }} + {{- end }} + {{- with $blockedQuery.Types }} + types: {{ . }} + {{- end }} + {{- with $blockedQuery.Hash }} + hash: {{ . }} + {{- end }} + {{- end }} + {{- end}} {{- end -}} {{- with $spec.Retention }} retention_period: {{ .Days }}d diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 5829270244322..110bce9ee8df3 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -41,7 +41,7 @@ type Options struct { } type LokiOverrides struct { - Limits lokiv1.LimitsTemplateSpec + Limits lokiv1.PerTenantLimitsTemplateSpec Ruler RulerOverrides } diff --git a/operator/internal/manifests/options_test.go b/operator/internal/manifests/options_test.go index 48e9853f7870a..bf297adc650ff 100644 --- a/operator/internal/manifests/options_test.go +++ b/operator/internal/manifests/options_test.go @@ -60,15 +60,19 @@ func TestNewTimeoutConfig_ReturnsCustomConfig_WhenLimitsSpecNotEmpty_UseMaxTenan QueryTimeout: "10m", }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "tenant-a": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "10m", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "10m", + }, }, }, "tenant-b": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "20m", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "20m", + }, }, }, }, @@ -99,15 +103,19 @@ func TestNewTimeoutConfig_ReturnsCustomConfig_WhenTenantLimitsSpecOnly_ReturnsUs s := lokiv1.LokiStack{ Spec: lokiv1.LokiStackSpec{ Limits: &lokiv1.LimitsSpec{ - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "tenant-a": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "10m", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "10m", + }, }, }, "tenant-b": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "20m", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "20m", + }, }, }, }, @@ -160,15 +168,19 @@ func TestNewTimeoutConfig_ReturnsDefaults_WhenTenantQueryTimeoutParseError(t *te QueryTimeout: "10m", }, }, - Tenants: map[string]lokiv1.LimitsTemplateSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "tenant-a": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "invalid", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "invalid", + }, }, }, "tenant-b": { - QueryLimits: &lokiv1.QueryLimitSpec{ - QueryTimeout: "20m", + QueryLimits: &lokiv1.PerTenantQueryLimitSpec{ + QueryLimitSpec: lokiv1.QueryLimitSpec{ + QueryTimeout: "20m", + }, }, }, }, diff --git a/operator/internal/metrics/metrics.go b/operator/internal/metrics/metrics.go index fcf82ddca5d69..3c994f13c61ef 100644 --- a/operator/internal/metrics/metrics.go +++ b/operator/internal/metrics/metrics.go @@ -142,7 +142,7 @@ func boolValue(value bool) float64 { return 0 } -func streamRate(tenantLimits map[string]lokiv1.LimitsTemplateSpec, ingesters int32) float64 { +func streamRate(tenantLimits map[string]lokiv1.PerTenantLimitsTemplateSpec, ingesters int32) float64 { var tenants, tenantStreamLimit int32 = 0, 0 for _, tenant := range tenantLimits {