diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5adea5d2a02..60b9e3dc5e2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ * [11657](https://github.com/grafana/loki/pull/11657) **ashwanthgoli** Log results cache: compose empty response based on the request being served to avoid returning incorrect limit or direction. * [11587](https://github.com/grafana/loki/pull/11587) **trevorwhitney** Fix semantics of label parsing logic of metrics and logs queries. Both only parse the first label if multiple extractions into the same label are requested. * [11776](https://github.com/grafana/loki/pull/11776) **ashwanthgoli** Background Cache: Fixes a bug that is causing the background queue size to be incremented twice for each enqueued item. +* [11921](https://github.com/grafana/loki/pull/11921) **paul1r**: Parsing: String array elements were not being parsed correctly in JSON processing ##### Changes diff --git a/clients/pkg/promtail/targets/cloudflare/target.go b/clients/pkg/promtail/targets/cloudflare/target.go index b64e33da4bc29..19d1f18758273 100644 --- a/clients/pkg/promtail/targets/cloudflare/target.go +++ b/clients/pkg/promtail/targets/cloudflare/target.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "github.com/buger/jsonparser" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/cloudflare-go" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/concurrency" "github.com/grafana/dskit/multierror" + "github.com/grafana/jsonparser" "github.com/prometheus/common/model" "go.uber.org/atomic" diff --git a/go.mod b/go.mod index 87ea0fd075852..6235582406d50 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/aws/aws-sdk-go v1.44.321 github.com/baidubce/bce-sdk-go v0.9.141 github.com/bmatcuk/doublestar v1.3.4 - github.com/buger/jsonparser v1.1.1 github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.2.0 @@ -124,6 +123,7 @@ require ( github.com/efficientgo/core v1.0.0-rc.2 github.com/fsnotify/fsnotify v1.6.0 github.com/gogo/googleapis v1.4.0 + github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 github.com/heroku/x v0.0.61 github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b diff --git a/go.sum b/go.sum index dd756d74f7c69..8ab729e928055 100644 --- a/go.sum +++ b/go.sum @@ -390,8 +390,6 @@ github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/caddyserver/caddy v1.0.4/go.mod h1:uruyfVsyMcDb3IOzSKsi1x0wOjy1my/PxOSTcD+24jM= @@ -1003,6 +1001,8 @@ github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 h1:xLuzPoOzdfNb/RF/I github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85/go.mod h1:crI9WX6p0IhrqB+DqIUHulRW853PaNFf7o4UprV//3I= github.com/grafana/gomemcache v0.0.0-20231204155601-7de47a8c3cb0 h1:aLBiDMjTtXx2800iCIp+8kdjIlvGX0MF/zICQMQO2qU= github.com/grafana/gomemcache v0.0.0-20231204155601-7de47a8c3cb0/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= +github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d h1:YwbJJ/PrVWVdnR+j/EAVuazdeP+Za5qbiH1Vlr+wFXs= +github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d/go.mod h1:796sq+UcONnSlzA3RtlBZ+b/hrerkZXiEmO8oMjyRwY= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= diff --git a/integration/client/client.go b/integration/client/client.go index 2e5a86aa6b3de..1ad94fd0edbb6 100644 --- a/integration/client/client.go +++ b/integration/client/client.go @@ -14,9 +14,9 @@ import ( "strings" "time" - "github.com/buger/jsonparser" "github.com/gorilla/websocket" "github.com/grafana/dskit/user" + "github.com/grafana/jsonparser" "github.com/prometheus/common/config" "github.com/prometheus/prometheus/model/labels" "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/operator/.bingo/go.mod b/operator/.bingo/go.mod index 610249af0b0b0..3aa5b7c946f5d 100644 --- a/operator/.bingo/go.mod +++ b/operator/.bingo/go.mod @@ -1 +1 @@ -module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. \ No newline at end of file +module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. diff --git a/pkg/loghttp/entry.go b/pkg/loghttp/entry.go index 2a55ac9ecd285..0529bf536a2d5 100644 --- a/pkg/loghttp/entry.go +++ b/pkg/loghttp/entry.go @@ -6,7 +6,7 @@ import ( "time" "unsafe" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" jsoniter "github.com/json-iterator/go" "github.com/modern-go/reflect2" "github.com/prometheus/prometheus/model/labels" diff --git a/pkg/loghttp/labels.go b/pkg/loghttp/labels.go index b15a94ab23414..98bad4e957869 100644 --- a/pkg/loghttp/labels.go +++ b/pkg/loghttp/labels.go @@ -6,8 +6,8 @@ import ( "strconv" "strings" - "github.com/buger/jsonparser" "github.com/gorilla/mux" + "github.com/grafana/jsonparser" "github.com/grafana/loki/pkg/logproto" ) diff --git a/pkg/loghttp/query.go b/pkg/loghttp/query.go index 617754393538c..854ccd5ae7116 100644 --- a/pkg/loghttp/query.go +++ b/pkg/loghttp/query.go @@ -8,7 +8,7 @@ import ( "time" "unsafe" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" json "github.com/json-iterator/go" "github.com/prometheus/common/model" diff --git a/pkg/logql/log/parser.go b/pkg/logql/log/parser.go index c03e7c91cb960..90d4a4bebf8ab 100644 --- a/pkg/logql/log/parser.go +++ b/pkg/logql/log/parser.go @@ -6,7 +6,7 @@ import ( "fmt" "unicode/utf8" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" "github.com/grafana/loki/pkg/logql/log/jsonexpr" "github.com/grafana/loki/pkg/logql/log/logfmt" diff --git a/pkg/logql/log/parser_test.go b/pkg/logql/log/parser_test.go index bd57603ab8084..f8cf6373a152f 100644 --- a/pkg/logql/log/parser_test.go +++ b/pkg/logql/log/parser_test.go @@ -237,7 +237,7 @@ func (p *fakeParseHints) ShouldContinueParsingLine(_ string, _ *LabelsBuilder) b } func TestJSONExpressionParser(t *testing.T) { - testLine := []byte(`{"app":"foo","field with space":"value","field with ÜFT8👌":"value","null_field":null,"bool_field":false,"namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar", "params": [1,2,3]}}}`) + testLine := []byte(`{"app":"foo","field with space":"value","field with ÜFT8👌":"value","null_field":null,"bool_field":false,"namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar", "params": [1,2,3,"string_value"]}}}`) tests := []struct { name string @@ -340,6 +340,16 @@ func TestJSONExpressionParser(t *testing.T) { labels.FromStrings("param", "1"), NoParserHints(), }, + { + "array string element", + testLine, + []LabelExtractionExpr{ + NewLabelExtractionExpr("param", `pod.deployment.params[3]`), + }, + labels.EmptyLabels(), + labels.FromStrings("param", "string_value"), + NoParserHints(), + }, { "full array", testLine, @@ -347,7 +357,7 @@ func TestJSONExpressionParser(t *testing.T) { NewLabelExtractionExpr("params", `pod.deployment.params`), }, labels.EmptyLabels(), - labels.FromStrings("params", "[1,2,3]"), + labels.FromStrings("params", `[1,2,3,"string_value"]`), NoParserHints(), }, { @@ -357,7 +367,7 @@ func TestJSONExpressionParser(t *testing.T) { NewLabelExtractionExpr("deployment", `pod.deployment`), }, labels.EmptyLabels(), - labels.FromStrings("deployment", `{"ref":"foobar", "params": [1,2,3]}`), + labels.FromStrings("deployment", `{"ref":"foobar", "params": [1,2,3,"string_value"]}`), NoParserHints(), }, { diff --git a/pkg/querier/queryrange/extensions.go b/pkg/querier/queryrange/extensions.go index b8a0ca7f41935..75d4ce2cb4edd 100644 --- a/pkg/querier/queryrange/extensions.go +++ b/pkg/querier/queryrange/extensions.go @@ -3,7 +3,7 @@ package queryrange import ( "fmt" - "github.com/buger/jsonparser" + "github.com/grafana/jsonparser" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" diff --git a/vendor/github.com/buger/jsonparser/.travis.yml b/vendor/github.com/buger/jsonparser/.travis.yml deleted file mode 100644 index dbfb7cf988305..0000000000000 --- a/vendor/github.com/buger/jsonparser/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: go -arch: - - amd64 - - ppc64le -go: - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - 1.11.x -script: go test -v ./. diff --git a/vendor/github.com/buger/jsonparser/.gitignore b/vendor/github.com/grafana/jsonparser/.gitignore similarity index 100% rename from vendor/github.com/buger/jsonparser/.gitignore rename to vendor/github.com/grafana/jsonparser/.gitignore diff --git a/vendor/github.com/grafana/jsonparser/.travis.yml b/vendor/github.com/grafana/jsonparser/.travis.yml new file mode 100644 index 0000000000000..56f9c9c421234 --- /dev/null +++ b/vendor/github.com/grafana/jsonparser/.travis.yml @@ -0,0 +1,12 @@ +language: go +arch: + - amd64 + - ppc64le +go: + - 1.13.x + - 1.14.x + - 1.15.x + - 1.16.x + - 1.17.x + - 1.18.x +script: go test -v ./. diff --git a/vendor/github.com/buger/jsonparser/Dockerfile b/vendor/github.com/grafana/jsonparser/Dockerfile similarity index 100% rename from vendor/github.com/buger/jsonparser/Dockerfile rename to vendor/github.com/grafana/jsonparser/Dockerfile diff --git a/vendor/github.com/buger/jsonparser/LICENSE b/vendor/github.com/grafana/jsonparser/LICENSE similarity index 100% rename from vendor/github.com/buger/jsonparser/LICENSE rename to vendor/github.com/grafana/jsonparser/LICENSE diff --git a/vendor/github.com/buger/jsonparser/Makefile b/vendor/github.com/grafana/jsonparser/Makefile similarity index 100% rename from vendor/github.com/buger/jsonparser/Makefile rename to vendor/github.com/grafana/jsonparser/Makefile diff --git a/vendor/github.com/buger/jsonparser/README.md b/vendor/github.com/grafana/jsonparser/README.md similarity index 99% rename from vendor/github.com/buger/jsonparser/README.md rename to vendor/github.com/grafana/jsonparser/README.md index d7e0ec397affe..0b2f1fb037123 100644 --- a/vendor/github.com/buger/jsonparser/README.md +++ b/vendor/github.com/grafana/jsonparser/README.md @@ -90,10 +90,6 @@ jsonparser.EachKey(data, func(idx int, value []byte, vt jsonparser.ValueType, er // For more information see docs below ``` -## Need to speedup your app? - -I'm available for consulting and can help you push your app performance to the limits. Ping me at: leonsbox@gmail.com. - ## Reference Library API is really simple. You just need the `Get` method to perform any operation. The rest is just helpers around it. diff --git a/vendor/github.com/buger/jsonparser/bytes.go b/vendor/github.com/grafana/jsonparser/bytes.go similarity index 50% rename from vendor/github.com/buger/jsonparser/bytes.go rename to vendor/github.com/grafana/jsonparser/bytes.go index 0bb0ff39562cb..9d6e701f58364 100644 --- a/vendor/github.com/buger/jsonparser/bytes.go +++ b/vendor/github.com/grafana/jsonparser/bytes.go @@ -1,11 +1,8 @@ package jsonparser -import ( - bio "bytes" -) - -// minInt64 '-9223372036854775808' is the smallest representable number in int64 -const minInt64 = `9223372036854775808` +const absMinInt64 = 1 << 63 +const maxInt64 = 1<<63 - 1 +const maxUint64 = 1<<64 - 1 // About 2x faster then strconv.ParseInt because it only supports base 10, which is enough for JSON func parseInt(bytes []byte) (v int64, ok bool, overflow bool) { @@ -19,29 +16,32 @@ func parseInt(bytes []byte) (v int64, ok bool, overflow bool) { bytes = bytes[1:] } - var b int64 = 0 + var n uint64 = 0 for _, c := range bytes { - if c >= '0' && c <= '9' { - b = (10 * v) + int64(c-'0') - } else { + if c < '0' || c > '9' { return 0, false, false } - if overflow = (b < v); overflow { - break + if n > maxUint64/10 { + return 0, false, true + } + n *= 10 + n1 := n + uint64(c-'0') + if n1 < n { + return 0, false, true } - v = b + n = n1 } - if overflow { - if neg && bio.Equal(bytes, []byte(minInt64)) { - return b, true, false + if n > maxInt64 { + if neg && n == absMinInt64 { + return -absMinInt64, true, false } return 0, false, true } if neg { - return -v, true, false + return -int64(n), true, false } else { - return v, true, false + return int64(n), true, false } } diff --git a/vendor/github.com/buger/jsonparser/bytes_safe.go b/vendor/github.com/grafana/jsonparser/bytes_safe.go similarity index 100% rename from vendor/github.com/buger/jsonparser/bytes_safe.go rename to vendor/github.com/grafana/jsonparser/bytes_safe.go diff --git a/vendor/github.com/buger/jsonparser/bytes_unsafe.go b/vendor/github.com/grafana/jsonparser/bytes_unsafe.go similarity index 100% rename from vendor/github.com/buger/jsonparser/bytes_unsafe.go rename to vendor/github.com/grafana/jsonparser/bytes_unsafe.go diff --git a/vendor/github.com/buger/jsonparser/escape.go b/vendor/github.com/grafana/jsonparser/escape.go similarity index 100% rename from vendor/github.com/buger/jsonparser/escape.go rename to vendor/github.com/grafana/jsonparser/escape.go diff --git a/vendor/github.com/buger/jsonparser/fuzz.go b/vendor/github.com/grafana/jsonparser/fuzz.go similarity index 100% rename from vendor/github.com/buger/jsonparser/fuzz.go rename to vendor/github.com/grafana/jsonparser/fuzz.go diff --git a/vendor/github.com/buger/jsonparser/oss-fuzz-build.sh b/vendor/github.com/grafana/jsonparser/oss-fuzz-build.sh similarity index 100% rename from vendor/github.com/buger/jsonparser/oss-fuzz-build.sh rename to vendor/github.com/grafana/jsonparser/oss-fuzz-build.sh diff --git a/vendor/github.com/buger/jsonparser/parser.go b/vendor/github.com/grafana/jsonparser/parser.go similarity index 96% rename from vendor/github.com/buger/jsonparser/parser.go rename to vendor/github.com/grafana/jsonparser/parser.go index 14b80bc4838c5..5df2a463dcee3 100644 --- a/vendor/github.com/buger/jsonparser/parser.go +++ b/vendor/github.com/grafana/jsonparser/parser.go @@ -18,6 +18,7 @@ var ( MalformedValueError = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol") OverflowIntegerError = errors.New("Value is number, but overflowed while parsing") MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string") + NullValueError = errors.New("Value is null") ) // How much stack space to allocate for unescaping JSON strings; if a string longer @@ -49,10 +50,13 @@ func findTokenStart(data []byte, token byte) int { } func findKeyStart(data []byte, key string) (int, error) { - i := 0 + i := nextToken(data) + if i == -1 { + return i, KeyPathNotFoundError + } ln := len(data) - if ln > 0 && (data[0] == '{' || data[0] == '[') { - i = 1 + if ln > 0 && (data[i] == '{' || data[i] == '[') { + i += 1 } var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings @@ -308,7 +312,7 @@ func searchKeys(data []byte, keys ...string) int { case '[': // If we want to get array element by index if keyLevel == level && keys[level][0] == '[' { - var keyLen = len(keys[level]) + keyLen := len(keys[level]) if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { return -1 } @@ -319,7 +323,7 @@ func searchKeys(data []byte, keys ...string) int { var curIdx int var valueFound []byte var valueOffset int - var curI = i + curI := i ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { if curIdx == aIdx { valueFound = value @@ -374,12 +378,19 @@ func sameTree(p1, p2 []string) bool { return true } +const stackArraySize = 128 + func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int { var x struct{} - pathFlags := make([]bool, len(paths)) var level, pathsMatched, i int ln := len(data) + pathFlags := make([]bool, stackArraySize)[:] + if len(paths) > cap(pathFlags) { + pathFlags = make([]bool, len(paths))[:] + } + pathFlags = pathFlags[0:len(paths)] + var maxPath int for _, p := range paths { if len(p) > maxPath { @@ -387,7 +398,11 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str } } - pathsBuf := make([]string, maxPath) + pathsBuf := make([]string, stackArraySize)[:] + if maxPath > cap(pathsBuf) { + pathsBuf = make([]string, maxPath)[:] + } + pathsBuf = pathsBuf[0:maxPath] for i < ln { switch data[i] { @@ -484,7 +499,12 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str case '[': var ok bool arrIdxFlags := make(map[int]struct{}) - pIdxFlags := make([]bool, len(paths)) + + pIdxFlags := make([]bool, stackArraySize)[:] + if len(paths) > cap(pIdxFlags) { + pIdxFlags = make([]bool, len(paths))[:] + } + pIdxFlags = pIdxFlags[0:len(paths)] if level < 0 { cb(-1, nil, Unknown, MalformedJsonError) @@ -519,8 +539,13 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str pathFlags[pi] = true if of != -1 { - v, dt, _, e := Get(value[of:]) - cb(pi, v, dt, e) + if dataType == String { + // the double-quotes were stripped, so we cannot call Get again. + cb(pi, value[of:], dataType, nil) + } else { + v, dt, _, e := Get(value[of:]) + cb(pi, v, dt, e) + } } } } @@ -662,7 +687,6 @@ func calcAllocateSpace(keys []string, setValue []byte, comma, object bool) int { } } - lk += len(setValue) for i := 1; i < len(keys); i++ { if string(keys[i][0]) == "[" { @@ -1178,6 +1202,9 @@ func GetString(data []byte, keys ...string) (val string, err error) { } if t != String { + if t == Null { + return "", NullValueError + } return "", fmt.Errorf("Value is not a string: %s", string(v)) } @@ -1200,6 +1227,9 @@ func GetFloat(data []byte, keys ...string) (val float64, err error) { } if t != Number { + if t == Null { + return 0, NullValueError + } return 0, fmt.Errorf("Value is not a number: %s", string(v)) } @@ -1216,6 +1246,9 @@ func GetInt(data []byte, keys ...string) (val int64, err error) { } if t != Number { + if t == Null { + return 0, NullValueError + } return 0, fmt.Errorf("Value is not a number: %s", string(v)) } @@ -1233,6 +1266,9 @@ func GetBoolean(data []byte, keys ...string) (val bool, err error) { } if t != Boolean { + if t == Null { + return false, NullValueError + } return false, fmt.Errorf("Value is not a boolean: %s", string(v)) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 371c205834933..e7cbf47887d61 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -420,9 +420,6 @@ github.com/beorn7/perks/quantile # github.com/bmatcuk/doublestar v1.3.4 ## explicit; go 1.12 github.com/bmatcuk/doublestar -# github.com/buger/jsonparser v1.1.1 -## explicit; go 1.13 -github.com/buger/jsonparser # github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b ## explicit github.com/c2h5oh/datasize @@ -905,6 +902,9 @@ github.com/grafana/go-gelf/v2/gelf # github.com/grafana/gomemcache v0.0.0-20231204155601-7de47a8c3cb0 ## explicit; go 1.18 github.com/grafana/gomemcache/memcache +# github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d +## explicit; go 1.13 +github.com/grafana/jsonparser # github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 => ./pkg/push ## explicit; go 1.19 github.com/grafana/loki/pkg/push