From 0a2bb85aefe2db9f941e47e819735f9c797674a6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:08:18 +0000 Subject: [PATCH 1/5] fix(deps): update module gopkg.in/yaml.v2 to v3 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 24bdc41341c7d..7a74f2d26e8cb 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,6 @@ require ( golang.org/x/time v0.8.0 google.golang.org/api v0.206.0 google.golang.org/grpc v1.68.0 - gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/klog v1.0.0 ) @@ -149,6 +148,7 @@ require ( golang.org/x/oauth2 v0.24.0 golang.org/x/text v0.20.0 google.golang.org/protobuf v1.35.2 + gopkg.in/yaml.v2 v2.4.0 gotest.tools v2.2.0+incompatible k8s.io/apimachinery v0.31.2 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 From 029af7ced50c12c755e617a139bc945e726fb77c Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Wed, 20 Nov 2024 12:42:29 -0500 Subject: [PATCH 2/5] First pass, need to figure out what to do with yaml v2->v3 differences --- clients/cmd/docker-driver/config.go | 24 +++++++++++++++---- clients/pkg/logentry/stages/json_test.go | 2 +- clients/pkg/logentry/stages/logfmt_test.go | 2 +- clients/pkg/logentry/stages/regex_test.go | 2 +- clients/pkg/logentry/stages/replace_test.go | 2 +- clients/pkg/logentry/stages/stage.go | 2 +- clients/pkg/promtail/client/config_test.go | 2 +- clients/pkg/promtail/client/logger.go | 2 +- clients/pkg/promtail/config/config.go | 2 +- clients/pkg/promtail/config/config_test.go | 2 +- clients/pkg/promtail/positions/positions.go | 12 +++++++--- .../positions/write_positions_unix.go | 2 +- .../positions/write_positions_windows.go | 2 +- .../scrapeconfig/scrapeconfig_test.go | 2 +- .../targets/journal/journaltarget_test.go | 2 +- .../stdin/stdin_target_manager_test.go | 2 +- .../targets/syslog/syslogtarget_test.go | 2 +- go.mod | 2 +- integration/cluster/cluster.go | 2 +- integration/util/merger.go | 2 +- pkg/logcli/query/query.go | 4 ++-- pkg/loghttp/push/otlp_config_test.go | 8 +++++-- pkg/loki/config_handler.go | 2 +- pkg/loki/runtime_config.go | 4 ++-- pkg/querier/queryrange/limits_test.go | 2 +- .../queryrange/split_by_interval_test.go | 2 +- pkg/ruler/base/ruler_test.go | 2 +- pkg/ruler/config.go | 2 +- pkg/ruler/registry.go | 2 +- pkg/ruler/storage/instance/instance.go | 10 ++++---- pkg/ruler/storage/instance/marshal.go | 4 ++-- pkg/ruler/storage/instance/marshal_test.go | 2 +- pkg/ruler/storage/util/compare_yaml.go | 2 +- pkg/sizing/http.go | 2 +- pkg/storage/bucket/client_test.go | 2 +- pkg/storage/bucket/http/config_test.go | 2 +- .../client/aws/s3_storage_client_test.go | 2 +- .../baidubce/bos_storage_client_test.go | 2 +- pkg/storage/config/bench_test.go | 2 +- pkg/storage/config/schema_config.go | 4 ++-- pkg/storage/config/schema_config_test.go | 2 +- pkg/util/cfg/files.go | 8 +++++-- pkg/util/config.go | 2 +- pkg/util/flagext/bytesize_test.go | 2 +- pkg/util/flagext/labelset.go | 2 +- pkg/util/http.go | 2 +- pkg/util/http_test.go | 2 +- pkg/validation/limits.go | 2 +- pkg/validation/limits_test.go | 8 +++++-- production/helm/loki/test/config_test.go | 2 +- tools/tsdb/tsdb-map/main.go | 2 +- 51 files changed, 101 insertions(+), 67 deletions(-) diff --git a/clients/cmd/docker-driver/config.go b/clients/cmd/docker-driver/config.go index d53117ca4872b..19cfac7042d31 100644 --- a/clients/cmd/docker-driver/config.go +++ b/clients/cmd/docker-driver/config.go @@ -3,6 +3,7 @@ package main import ( "bytes" "fmt" + "io" "net/url" "os" "strconv" @@ -17,7 +18,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/logentry/stages" "github.com/grafana/loki/v3/clients/pkg/promtail/client" @@ -319,7 +320,11 @@ func parsePipeline(logCtx logger.Info) (PipelineConfig, error) { } } if okString { - if err := yaml.UnmarshalStrict([]byte(pipelineString), &pipeline.PipelineStages); err != nil { + decoder := yaml.NewDecoder(bytes.NewReader([]byte(pipelineString))) + decoder.KnownFields(true) + + err := decoder.Decode(&pipeline) + if err != nil && !errors.Is(err, io.EOF) { return pipeline, err } } @@ -363,7 +368,11 @@ func parseInt(key string, logCtx logger.Info, set func(i int)) error { func relabelConfig(config string, lbs model.LabelSet) (model.LabelSet, error) { relabelConfig := make([]*relabel.Config, 0) - if err := yaml.UnmarshalStrict([]byte(config), &relabelConfig); err != nil { + decoder := yaml.NewDecoder(bytes.NewReader([]byte(config))) + decoder.KnownFields(true) + + err := decoder.Decode(&relabelConfig) + if err != nil && !errors.Is(err, io.EOF) { return nil, err } relabed, _ := relabel.Process(labels.FromMap(util.ModelLabelSetToMap(lbs)), relabelConfig...) @@ -389,5 +398,12 @@ func loadConfig(filename string, cfg interface{}) error { return errors.Wrap(err, "Error reading config file") } - return yaml.UnmarshalStrict(buf, cfg) + decoder := yaml.NewDecoder(bytes.NewReader(buf)) + decoder.KnownFields(true) + + err = decoder.Decode(cfg) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + return nil } diff --git a/clients/pkg/logentry/stages/json_test.go b/clients/pkg/logentry/stages/json_test.go index 9e0d2dffaea4c..82ac43afc5df8 100644 --- a/clients/pkg/logentry/stages/json_test.go +++ b/clients/pkg/logentry/stages/json_test.go @@ -8,7 +8,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" util_log "github.com/grafana/loki/v3/pkg/util/log" ) diff --git a/clients/pkg/logentry/stages/logfmt_test.go b/clients/pkg/logentry/stages/logfmt_test.go index 1406aa080cf88..de88bfeeef4ec 100644 --- a/clients/pkg/logentry/stages/logfmt_test.go +++ b/clients/pkg/logentry/stages/logfmt_test.go @@ -7,7 +7,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" util_log "github.com/grafana/loki/v3/pkg/util/log" ) diff --git a/clients/pkg/logentry/stages/regex_test.go b/clients/pkg/logentry/stages/regex_test.go index 362c5990575cb..742f2cba2260c 100644 --- a/clients/pkg/logentry/stages/regex_test.go +++ b/clients/pkg/logentry/stages/regex_test.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" util_log "github.com/grafana/loki/v3/pkg/util/log" ) diff --git a/clients/pkg/logentry/stages/replace_test.go b/clients/pkg/logentry/stages/replace_test.go index 6e583f19b8b06..ea78cf27d9150 100644 --- a/clients/pkg/logentry/stages/replace_test.go +++ b/clients/pkg/logentry/stages/replace_test.go @@ -8,7 +8,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" util_log "github.com/grafana/loki/v3/pkg/util/log" ) diff --git a/clients/pkg/logentry/stages/stage.go b/clients/pkg/logentry/stages/stage.go index 82e4e508d8237..025cb81ecf320 100644 --- a/clients/pkg/logentry/stages/stage.go +++ b/clients/pkg/logentry/stages/stage.go @@ -10,7 +10,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/promtail/api" ) diff --git a/clients/pkg/promtail/client/config_test.go b/clients/pkg/promtail/client/config_test.go index cb1476f1a2a0a..9a4c3e91458ec 100644 --- a/clients/pkg/promtail/client/config_test.go +++ b/clients/pkg/promtail/client/config_test.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/common/config" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) var clientConfig = Config{} diff --git a/clients/pkg/promtail/client/logger.go b/clients/pkg/promtail/client/logger.go index ba20055a0d94b..e81bcc8f99ef8 100644 --- a/clients/pkg/promtail/client/logger.go +++ b/clients/pkg/promtail/client/logger.go @@ -10,7 +10,7 @@ import ( "github.com/fatih/color" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/promtail/api" "github.com/grafana/loki/v3/clients/pkg/promtail/limit" diff --git a/clients/pkg/promtail/config/config.go b/clients/pkg/promtail/config/config.go index 7e0e2b63fe173..bc2257ab72925 100644 --- a/clients/pkg/promtail/config/config.go +++ b/clients/pkg/promtail/config/config.go @@ -6,7 +6,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/promtail/client" "github.com/grafana/loki/v3/clients/pkg/promtail/limit" diff --git a/clients/pkg/promtail/config/config_test.go b/clients/pkg/promtail/config/config_test.go index fe197044b683f..0b60d28a13f82 100644 --- a/clients/pkg/promtail/config/config_test.go +++ b/clients/pkg/promtail/config/config_test.go @@ -9,7 +9,7 @@ import ( dskitflagext "github.com/grafana/dskit/flagext" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/promtail/client" diff --git a/clients/pkg/promtail/positions/positions.go b/clients/pkg/promtail/positions/positions.go index 64c4bb6315327..9fba7d05dba37 100644 --- a/clients/pkg/promtail/positions/positions.go +++ b/clients/pkg/promtail/positions/positions.go @@ -1,8 +1,11 @@ package positions import ( + "bytes" + "errors" "flag" "fmt" + "io" "os" "path/filepath" "strconv" @@ -12,7 +15,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" ) const ( @@ -224,8 +227,11 @@ func readPositionsFile(cfg Config, logger log.Logger) (map[string]string, error) } var p File - err = yaml.UnmarshalStrict(buf, &p) - if err != nil { + decoder := yaml.NewDecoder(bytes.NewReader(buf)) + decoder.KnownFields(true) + + err = decoder.Decode(p) + if err != nil && !errors.Is(err, io.EOF) { // return empty if cfg option enabled if cfg.IgnoreInvalidYaml { level.Debug(logger).Log("msg", "ignoring invalid positions file", "file", cleanfn, "error", err) diff --git a/clients/pkg/promtail/positions/write_positions_unix.go b/clients/pkg/promtail/positions/write_positions_unix.go index 9a4e122b5ed7a..e90f637b93f0f 100644 --- a/clients/pkg/promtail/positions/write_positions_unix.go +++ b/clients/pkg/promtail/positions/write_positions_unix.go @@ -8,7 +8,7 @@ import ( "path/filepath" renameio "github.com/google/renameio/v2" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" ) func writePositionFile(filename string, positions map[string]string) error { diff --git a/clients/pkg/promtail/positions/write_positions_windows.go b/clients/pkg/promtail/positions/write_positions_windows.go index c139376d49b0a..6d72252d8bdc7 100644 --- a/clients/pkg/promtail/positions/write_positions_windows.go +++ b/clients/pkg/promtail/positions/write_positions_windows.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" ) // writePositionFile is a fall back for Windows because renameio does not support Windows. diff --git a/clients/pkg/promtail/scrapeconfig/scrapeconfig_test.go b/clients/pkg/promtail/scrapeconfig/scrapeconfig_test.go index f8898d86aa591..bf78bd16a69ac 100644 --- a/clients/pkg/promtail/scrapeconfig/scrapeconfig_test.go +++ b/clients/pkg/promtail/scrapeconfig/scrapeconfig_test.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/prometheus/discovery/kubernetes" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // todo add full example. diff --git a/clients/pkg/promtail/targets/journal/journaltarget_test.go b/clients/pkg/promtail/targets/journal/journaltarget_test.go index 2aa141be610e3..b8e29bdc0dbfc 100644 --- a/clients/pkg/promtail/targets/journal/journaltarget_test.go +++ b/clients/pkg/promtail/targets/journal/journaltarget_test.go @@ -18,7 +18,7 @@ import ( "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/promtail/client/fake" "github.com/grafana/loki/v3/clients/pkg/promtail/positions" diff --git a/clients/pkg/promtail/targets/stdin/stdin_target_manager_test.go b/clients/pkg/promtail/targets/stdin/stdin_target_manager_test.go index 8f2135f3aff32..6599c2e08087e 100644 --- a/clients/pkg/promtail/targets/stdin/stdin_target_manager_test.go +++ b/clients/pkg/promtail/targets/stdin/stdin_target_manager_test.go @@ -10,7 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/logentry/stages" "github.com/grafana/loki/v3/clients/pkg/promtail/api" diff --git a/clients/pkg/promtail/targets/syslog/syslogtarget_test.go b/clients/pkg/promtail/targets/syslog/syslogtarget_test.go index 7f379320f0c61..0a555206219c3 100644 --- a/clients/pkg/promtail/targets/syslog/syslogtarget_test.go +++ b/clients/pkg/promtail/targets/syslog/syslogtarget_test.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/clients/pkg/promtail/client/fake" "github.com/grafana/loki/v3/clients/pkg/promtail/scrapeconfig" diff --git a/go.mod b/go.mod index 7a74f2d26e8cb..e5283020e0ca4 100644 --- a/go.mod +++ b/go.mod @@ -148,7 +148,6 @@ require ( golang.org/x/oauth2 v0.24.0 golang.org/x/text v0.20.0 google.golang.org/protobuf v1.35.2 - gopkg.in/yaml.v2 v2.4.0 gotest.tools v2.2.0+incompatible k8s.io/apimachinery v0.31.2 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 @@ -191,6 +190,7 @@ require ( go.opentelemetry.io/otel/sdk v1.29.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( diff --git a/integration/cluster/cluster.go b/integration/cluster/cluster.go index 57bc182d0c8cb..e7959b081d378 100644 --- a/integration/cluster/cluster.go +++ b/integration/cluster/cluster.go @@ -21,7 +21,7 @@ import ( "github.com/grafana/dskit/multierror" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/integration/util" diff --git a/integration/util/merger.go b/integration/util/merger.go index dbd88a4205fc9..482f7c9315646 100644 --- a/integration/util/merger.go +++ b/integration/util/merger.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/imdario/mergo" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // YAMLMerger takes a set of given YAML fragments and merges them into a single YAML document. diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go index cb693a0125139..ccdedd5a092ca 100644 --- a/pkg/logcli/query/query.go +++ b/pkg/logcli/query/query.go @@ -14,7 +14,7 @@ import ( "github.com/grafana/dskit/user" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/logcli/client" "github.com/grafana/loki/v3/pkg/logcli/output" @@ -567,7 +567,7 @@ func LoadSchemaUsingObjectClient(oc chunk.ObjectClient, name string) (*config.Sc defer rdr.Close() decoder := yaml.NewDecoder(rdr) - decoder.SetStrict(true) + decoder.KnownFields(true) section := schemaConfigSection{} err = decoder.Decode(§ion) if err != nil { diff --git a/pkg/loghttp/push/otlp_config_test.go b/pkg/loghttp/push/otlp_config_test.go index de5b9b281a37c..485a7212c4f72 100644 --- a/pkg/loghttp/push/otlp_config_test.go +++ b/pkg/loghttp/push/otlp_config_test.go @@ -1,12 +1,13 @@ package push import ( + "bytes" "testing" "github.com/grafana/dskit/flagext" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) var defaultGlobalOTLPConfig = GlobalOTLPConfig{} @@ -153,7 +154,10 @@ log_attributes: } { t.Run(tc.name, func(t *testing.T) { cfg := OTLPConfig{} - err := yaml.UnmarshalStrict(tc.yamlConfig, &cfg) + decoder := yaml.NewDecoder(bytes.NewReader(tc.yamlConfig)) + decoder.KnownFields(true) + + err := decoder.Decode(cfg) if tc.expectedErr != nil { require.ErrorIs(t, err, tc.expectedErr) return diff --git a/pkg/loki/config_handler.go b/pkg/loki/config_handler.go index 48de706ff1b69..4365f9237b570 100644 --- a/pkg/loki/config_handler.go +++ b/pkg/loki/config_handler.go @@ -5,7 +5,7 @@ import ( "net/http" "reflect" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func yamlMarshalUnmarshal(in interface{}) (map[interface{}]interface{}, error) { diff --git a/pkg/loki/runtime_config.go b/pkg/loki/runtime_config.go index c2e72fbd4b78f..a11d313a57da7 100644 --- a/pkg/loki/runtime_config.go +++ b/pkg/loki/runtime_config.go @@ -7,7 +7,7 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/kv" "github.com/grafana/dskit/runtimeconfig" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/runtime" util_log "github.com/grafana/loki/v3/pkg/util/log" @@ -42,7 +42,7 @@ func loadRuntimeConfig(r io.Reader) (interface{}, error) { overrides := &runtimeConfigValues{} decoder := yaml.NewDecoder(r) - decoder.SetStrict(true) + decoder.KnownFields(true) if err := decoder.Decode(&overrides); err != nil { return nil, err } diff --git a/pkg/querier/queryrange/limits_test.go b/pkg/querier/queryrange/limits_test.go index d2ad3385b26ac..3fcd0bd9bf774 100644 --- a/pkg/querier/queryrange/limits_test.go +++ b/pkg/querier/queryrange/limits_test.go @@ -15,7 +15,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/stretchr/testify/require" "go.uber.org/atomic" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql/syntax" diff --git a/pkg/querier/queryrange/split_by_interval_test.go b/pkg/querier/queryrange/split_by_interval_test.go index de1b19be10450..439b872b84d5a 100644 --- a/pkg/querier/queryrange/split_by_interval_test.go +++ b/pkg/querier/queryrange/split_by_interval_test.go @@ -13,7 +13,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/loghttp" "github.com/grafana/loki/v3/pkg/logproto" diff --git a/pkg/ruler/base/ruler_test.go b/pkg/ruler/base/ruler_test.go index bdb437ed9279d..ac21946c9dd04 100644 --- a/pkg/ruler/base/ruler_test.go +++ b/pkg/ruler/base/ruler_test.go @@ -41,7 +41,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" "google.golang.org/grpc" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/dskit/tenant" diff --git a/pkg/ruler/config.go b/pkg/ruler/config.go index 205fdd2c6f1cf..8602e759cb4f4 100644 --- a/pkg/ruler/config.go +++ b/pkg/ruler/config.go @@ -7,7 +7,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/prometheus/config" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ruler "github.com/grafana/loki/v3/pkg/ruler/base" "github.com/grafana/loki/v3/pkg/ruler/storage/cleaner" diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 29297fcab5a74..0585e85ce3a49 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -23,7 +23,7 @@ import ( "github.com/prometheus/prometheus/model/metadata" "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/storage" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/ruler/storage/cleaner" "github.com/grafana/loki/v3/pkg/ruler/storage/instance" diff --git a/pkg/ruler/storage/instance/instance.go b/pkg/ruler/storage/instance/instance.go index ddd017664c976..96309b18e6dbb 100644 --- a/pkg/ruler/storage/instance/instance.go +++ b/pkg/ruler/storage/instance/instance.go @@ -25,7 +25,7 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb/wlog" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/ruler/storage/util" "github.com/grafana/loki/v3/pkg/ruler/storage/wal" @@ -88,13 +88,13 @@ func (c Config) MarshalYAML() (interface{}, error) { return nil, err } - // Use a yaml.MapSlice rather than a map[string]interface{} so + // Use a yaml.Node rather than a map[string]interface{} so // order of keys is retained compared to just calling MarshalConfig. - var m yaml.MapSlice - if err := yaml.Unmarshal(bb, &m); err != nil { + var n yaml.Node + if err := yaml.Unmarshal(bb, &n); err != nil { return nil, err } - return m, nil + return n, nil } // ApplyDefaults applies default configurations to the configuration to all diff --git a/pkg/ruler/storage/instance/marshal.go b/pkg/ruler/storage/instance/marshal.go index 2fb9e6abe2358..055ebbcdf8659 100644 --- a/pkg/ruler/storage/instance/marshal.go +++ b/pkg/ruler/storage/instance/marshal.go @@ -7,7 +7,7 @@ import ( "bytes" "io" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // UnmarshalConfig unmarshals an instance config from a reader based on a @@ -15,7 +15,7 @@ import ( func UnmarshalConfig(r io.Reader) (*Config, error) { var cfg Config dec := yaml.NewDecoder(r) - dec.SetStrict(true) + dec.KnownFields(true) err := dec.Decode(&cfg) return &cfg, err } diff --git a/pkg/ruler/storage/instance/marshal_test.go b/pkg/ruler/storage/instance/marshal_test.go index f47f31f8bdf99..76c69bdcb161e 100644 --- a/pkg/ruler/storage/instance/marshal_test.go +++ b/pkg/ruler/storage/instance/marshal_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func TestUnmarshalConfig_Valid(t *testing.T) { diff --git a/pkg/ruler/storage/util/compare_yaml.go b/pkg/ruler/storage/util/compare_yaml.go index 4ec346d638340..725e458cc76a6 100644 --- a/pkg/ruler/storage/util/compare_yaml.go +++ b/pkg/ruler/storage/util/compare_yaml.go @@ -6,7 +6,7 @@ package util import ( "bytes" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // CompareYAML marshals a and b to YAML and ensures that their contents are diff --git a/pkg/sizing/http.go b/pkg/sizing/http.go index f0b755af32ead..28371482c3de1 100644 --- a/pkg/sizing/http.go +++ b/pkg/sizing/http.go @@ -9,7 +9,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type Message struct { diff --git a/pkg/storage/bucket/client_test.go b/pkg/storage/bucket/client_test.go index a4bdb8f6e251c..81042622e266f 100644 --- a/pkg/storage/bucket/client_test.go +++ b/pkg/storage/bucket/client_test.go @@ -7,7 +7,7 @@ import ( "github.com/grafana/dskit/flagext" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" util_log "github.com/grafana/loki/v3/pkg/util/log" ) diff --git a/pkg/storage/bucket/http/config_test.go b/pkg/storage/bucket/http/config_test.go index 6ac47eb4a2572..a4cca47a72153 100644 --- a/pkg/storage/bucket/http/config_test.go +++ b/pkg/storage/bucket/http/config_test.go @@ -6,7 +6,7 @@ import ( "github.com/grafana/dskit/flagext" "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" ) // defaultConfig should match the default flag values defined in RegisterFlagsWithPrefix. diff --git a/pkg/storage/chunk/client/aws/s3_storage_client_test.go b/pkg/storage/chunk/client/aws/s3_storage_client_test.go index 38b0215b79136..5099c271bd53f 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client_test.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/flagext" diff --git a/pkg/storage/chunk/client/baidubce/bos_storage_client_test.go b/pkg/storage/chunk/client/baidubce/bos_storage_client_test.go index 8e68d9a1f2d47..0dd37e5cd5a3c 100644 --- a/pkg/storage/chunk/client/baidubce/bos_storage_client_test.go +++ b/pkg/storage/chunk/client/baidubce/bos_storage_client_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/dskit/flagext" ) diff --git a/pkg/storage/config/bench_test.go b/pkg/storage/config/bench_test.go index df5fb913b1f43..ac1a3b153a1b6 100644 --- a/pkg/storage/config/bench_test.go +++ b/pkg/storage/config/bench_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/storage/chunk" "github.com/grafana/loki/v3/pkg/storage/config" diff --git a/pkg/storage/config/schema_config.go b/pkg/storage/config/schema_config.go index 2c9a6a4605e84..a6ef5a886d8e9 100644 --- a/pkg/storage/config/schema_config.go +++ b/pkg/storage/config/schema_config.go @@ -15,7 +15,7 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/mtime" "github.com/prometheus/common/model" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/chunkenc" "github.com/grafana/loki/v3/pkg/logproto" @@ -286,7 +286,7 @@ func (cfg *SchemaConfig) loadFromFile() error { } decoder := yaml.NewDecoder(f) - decoder.SetStrict(true) + decoder.KnownFields(true) return decoder.Decode(&cfg) } diff --git a/pkg/storage/config/schema_config_test.go b/pkg/storage/config/schema_config_test.go index 8a8de6077e447..6164ce76b7487 100644 --- a/pkg/storage/config/schema_config_test.go +++ b/pkg/storage/config/schema_config_test.go @@ -10,7 +10,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/storage/chunk" diff --git a/pkg/util/cfg/files.go b/pkg/util/cfg/files.go index 6a3faa0f56f05..e386343d1164f 100644 --- a/pkg/util/cfg/files.go +++ b/pkg/util/cfg/files.go @@ -1,6 +1,7 @@ package cfg import ( + "bytes" "encoding/json" "flag" "fmt" @@ -10,7 +11,7 @@ import ( "github.com/drone/envsubst" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // JSON returns a Source that opens the supplied `.json` file and loads it. @@ -66,7 +67,10 @@ func YAML(f string, expandEnvVars bool, strict bool) Source { // dYAMLStrict returns a YAML source and allows dependency injection func dYAMLStrict(y []byte) Source { return func(dst Cloneable) error { - return yaml.UnmarshalStrict(y, dst) + decoder := yaml.NewDecoder(bytes.NewReader(y)) + decoder.KnownFields(true) + + return decoder.Decode(dst) } } diff --git a/pkg/util/config.go b/pkg/util/config.go index 89d586b37e13e..605d3a6564da4 100644 --- a/pkg/util/config.go +++ b/pkg/util/config.go @@ -8,7 +8,7 @@ import ( "github.com/go-kit/log/level" "github.com/prometheus/common/version" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" util_log "github.com/grafana/loki/v3/pkg/util/log" ) diff --git a/pkg/util/flagext/bytesize_test.go b/pkg/util/flagext/bytesize_test.go index 363645f9c1bba..068b6da05ae55 100644 --- a/pkg/util/flagext/bytesize_test.go +++ b/pkg/util/flagext/bytesize_test.go @@ -6,7 +6,7 @@ import ( "encoding/json" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func Test_ByteSize(t *testing.T) { diff --git a/pkg/util/flagext/labelset.go b/pkg/util/flagext/labelset.go index 79a72c07739e0..952a4054d4d8b 100644 --- a/pkg/util/flagext/labelset.go +++ b/pkg/util/flagext/labelset.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/prometheus/common/model" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/util" ) diff --git a/pkg/util/http.go b/pkg/util/http.go index 3fdfca6df24f1..4d38c35fecaf0 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -18,7 +18,7 @@ import ( "github.com/golang/snappy" "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) const messageSizeLargerErrFmt = "received message larger than max (%d vs %d)" diff --git a/pkg/util/http_test.go b/pkg/util/http_test.go index d032085db5028..2b49790f01c49 100644 --- a/pkg/util/http_test.go +++ b/pkg/util/http_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/util" diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 7387cc567c6a6..bae6efbbebc9f 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" "golang.org/x/time/rate" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/compactor/deletionmode" "github.com/grafana/loki/v3/pkg/compression" diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go index ce412305ceb79..4b6c1f1f5d821 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/validation/limits_test.go @@ -1,6 +1,7 @@ package validation import ( + "bytes" "encoding/json" "fmt" "reflect" @@ -10,7 +11,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/compactor/deletionmode" "github.com/grafana/loki/v3/pkg/compression" @@ -315,7 +316,10 @@ query_timeout: 5m t.Run(tc.desc, func(t *testing.T) { var out Limits - require.Nil(t, yaml.UnmarshalStrict([]byte(tc.yaml), &out)) + decoder := yaml.NewDecoder(bytes.NewReader([]byte(tc.yaml))) + decoder.KnownFields(true) + err := decoder.Decode(out) + require.NoError(t, err) require.Equal(t, tc.exp, out) }) } diff --git a/production/helm/loki/test/config_test.go b/production/helm/loki/test/config_test.go index 6926c7b2a85c2..cce11bfe5e744 100644 --- a/production/helm/loki/test/config_test.go +++ b/production/helm/loki/test/config_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type replicas struct { diff --git a/tools/tsdb/tsdb-map/main.go b/tools/tsdb/tsdb-map/main.go index 9f35b53fe48c6..67d7d57ed56e5 100644 --- a/tools/tsdb/tsdb-map/main.go +++ b/tools/tsdb/tsdb-map/main.go @@ -9,7 +9,7 @@ import ( "github.com/prometheus/common/model" "go.etcd.io/bbolt" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/grafana/loki/v3/pkg/compactor/retention" "github.com/grafana/loki/v3/pkg/storage/config" From 4ca0a18ae121f037e73ed2990190c80226debd10 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Thu, 28 Nov 2024 10:53:28 -0500 Subject: [PATCH 3/5] Some fixes for failing tests. Dates are parsed differerently --- integration/cluster/cluster.go | 10 +++++----- pkg/loghttp/push/otlp_config_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/cluster/cluster.go b/integration/cluster/cluster.go index 32d56b67edd1f..2766b08f5b670 100644 --- a/integration/cluster/cluster.go +++ b/integration/cluster/cluster.go @@ -27,7 +27,6 @@ import ( "github.com/grafana/loki/v3/pkg/loki" "github.com/grafana/loki/v3/pkg/storage" - "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/util/cfg" util_log "github.com/grafana/loki/v3/pkg/util/log" "github.com/grafana/loki/v3/pkg/validation" @@ -363,8 +362,9 @@ func (c *Component) writeConfig() error { func (c *Component) MergedConfig() ([]byte, error) { var sb bytes.Buffer - periodStart := config.DayTime{Time: c.cluster.initedAt.Add(-24 * time.Hour)} - additionalPeriodStart := config.DayTime{Time: c.cluster.initedAt.Add(-7 * 24 * time.Hour)} + // Quote the time strings to ensure they're treated as strings in YAML 1.2 + periodStart := fmt.Sprintf("%q", time.Unix(int64(c.cluster.initedAt)/1000, 0).Format("2006-01-02")) + additionalPeriodStart := fmt.Sprintf("%q", time.Unix(int64(c.cluster.initedAt)/1000, 0).Add(-6*24*time.Hour).Format("2006-01-02")) if err := configTemplate.Execute(&sb, map[string]interface{}{ "dataPath": c.dataPath, @@ -385,8 +385,8 @@ func (c *Component) MergedConfig() ([]byte, error) { var buf bytes.Buffer if err := template.Must(template.New("schema").Parse(periodCfg)). Execute(&buf, map[string]interface{}{ - "curPeriodStart": periodStart.String(), - "additionalPeriodStart": additionalPeriodStart.String(), + "curPeriodStart": periodStart, + "additionalPeriodStart": additionalPeriodStart, "schemaVer": c.cluster.schemaVer, }); err != nil { return nil, errors.New("error building schema_config") diff --git a/pkg/loghttp/push/otlp_config_test.go b/pkg/loghttp/push/otlp_config_test.go index 485a7212c4f72..1b0e070fc5f1e 100644 --- a/pkg/loghttp/push/otlp_config_test.go +++ b/pkg/loghttp/push/otlp_config_test.go @@ -157,7 +157,7 @@ log_attributes: decoder := yaml.NewDecoder(bytes.NewReader(tc.yamlConfig)) decoder.KnownFields(true) - err := decoder.Decode(cfg) + err := decoder.Decode(&cfg) if tc.expectedErr != nil { require.ErrorIs(t, err, tc.expectedErr) return From 7c33142d69f553d48afa7c1b26f8ada47fceaa8c Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Thu, 28 Nov 2024 11:20:08 -0500 Subject: [PATCH 4/5] Handle empty maps properly using old logic (not explicitly set) and new logic (declared as {}) --- pkg/validation/limits.go | 44 +++++++++++++++++++++++++++++++---- pkg/validation/limits_test.go | 20 +++++++++++++++- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index bdc9066bc5618..ce36a173b6973 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -478,6 +478,18 @@ func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error { return errors.Wrap(err, "cloning limits (unmarshaling)") } } + + // Check if we're getting an empty map for RulerRemoteWriteHeaders + var raw map[string]interface{} + if err := unmarshal(&raw); err != nil { + return err + } + + if v, ok := raw["ruler_remote_write_headers"]; ok && v == nil { + l.RulerRemoteWriteHeaders = OverwriteMarshalingStringMap{m: nil} + } + + // Now unmarshal the full config if err := unmarshal((*plain)(l)); err != nil { return err } @@ -1205,14 +1217,38 @@ func (sm OverwriteMarshalingStringMap) MarshalYAML() (interface{}, error) { return sm.m, nil } +// UnmarshalYAML implements yaml.Unmarshaler. func (sm *OverwriteMarshalingStringMap) UnmarshalYAML(unmarshal func(interface{}) error) error { - var def map[string]string + var raw interface{} + if err := unmarshal(&raw); err != nil { + return err + } - err := unmarshal(&def) - if err != nil { + // If we get a nil value or empty map, set a nil map + if raw == nil { + sm.m = nil + return nil + } + + // Check for empty map in different YAML representations + switch v := raw.(type) { + case map[interface{}]interface{}: + if len(v) == 0 { + sm.m = nil + return nil + } + case map[string]interface{}: + if len(v) == 0 { + sm.m = nil + return nil + } + } + + // Otherwise try to unmarshal as a map + var def map[string]string + if err := unmarshal(&def); err != nil { return err } sm.m = def - return nil } diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go index 4b6c1f1f5d821..7c8fb0c6476f0 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/validation/limits_test.go @@ -233,6 +233,24 @@ ruler_remote_write_headers: desc: "empty map overrides defaults", yaml: ` ruler_remote_write_headers: +`, + exp: Limits{ + DiscoverServiceName: []string{}, + LogLevelFields: []string{}, + // Rest from new defaults + StreamRetention: []StreamRetention{ + { + Period: model.Duration(24 * time.Hour), + Selector: `{a="b"}`, + }, + }, + OTLPConfig: defaultOTLPConfig, + }, + }, + { + desc: "explicitly set empty map overrides defaults", + yaml: ` +ruler_remote_write_headers: {} # Explicitly set an empty map `, exp: Limits{ DiscoverServiceName: []string{}, @@ -318,7 +336,7 @@ query_timeout: 5m var out Limits decoder := yaml.NewDecoder(bytes.NewReader([]byte(tc.yaml))) decoder.KnownFields(true) - err := decoder.Decode(out) + err := decoder.Decode(&out) require.NoError(t, err) require.Equal(t, tc.exp, out) }) From 6ce04a3427641e6d1661e98f23009a39fd4c30f0 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Mon, 2 Dec 2024 12:24:32 -0500 Subject: [PATCH 5/5] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c31d22e676af2..8b01a61df79be 100644 --- a/go.mod +++ b/go.mod @@ -146,6 +146,7 @@ require ( golang.org/x/oauth2 v0.24.0 golang.org/x/text v0.20.0 google.golang.org/protobuf v1.35.2 + gopkg.in/yaml.v2 v2.4.0 gotest.tools v2.2.0+incompatible k8s.io/apimachinery v0.31.3 k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 @@ -189,7 +190,6 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) require (