From 01f5883f16ca9ef27129c16c42f70b6737970e71 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 29 Aug 2024 08:23:43 +0200 Subject: [PATCH] Update grpc to v1.66.0 and switch to CodecV2 (#5900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gRPC v1.66.0 introduced CodecV2 that supports buffer pooling. The old V1 codecs no longer work (just the upgrade itself was causing panic, see https://github.com/grpc/grpc-go/commit/cfd14baa8264cbeebf6308a7b68333c8c2fc6e86#r145974307) This PR contains the following updates: * Upgrade `pkg/gogocodec/codec.go` to implement CodecV2 API. * Translate `mem.BufferSlice` into `[]byte` as needed (introduces a very small performance hit and increase in allocations) | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [google.golang.org/grpc](https://togithub.com/grpc/grpc-go) | `v1.65.0` -> `v1.66.0` | [![age](https://developer.mend.io/api/mc/badges/age/go/google.golang.org%2fgrpc/v1.66.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/go/google.golang.org%2fgrpc/v1.66.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/go/google.golang.org%2fgrpc/v1.65.0/v1.66.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/go/google.golang.org%2fgrpc/v1.65.0/v1.66.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | ## Performance differences Before ``` $ go test -benchmem -run=^$ -bench ^BenchmarkCodec github.com/jaegertracing/jaeger/pkg/gogocodec goos: darwin goarch: arm64 pkg: github.com/jaegertracing/jaeger/pkg/gogocodec cpu: Apple M1 Pro BenchmarkCodecUnmarshal25Spans-10 30015 39020 ns/op 108616 B/op 1082 allocs/op ``` After ``` goos: darwin goarch: arm64 pkg: github.com/jaegertracing/jaeger/pkg/gogocodec cpu: Apple M1 Pro BenchmarkCodecUnmarshal25Spans-10 29352 39752 ns/op 119496 B/op 1083 allocs/op ``` --- ### Release Notes
grpc/grpc-go (google.golang.org/grpc) ### [`v1.66.0`](https://togithub.com/grpc/grpc-go/releases/tag/v1.66.0): Release 1.66.0 [Compare Source](https://togithub.com/grpc/grpc-go/compare/v1.65.0...v1.66.0) ### New Features - metadata: stabilize `ValueFromIncomingContext` ([#​7368](https://togithub.com/grpc/grpc-go/issues/7368)) - Special Thanks: [@​KarthikReddyPuli](https://togithub.com/KarthikReddyPuli) - client: stabilize the `WaitForStateChange` and `GetState` methods, which were previously experimental. ([#​7425](https://togithub.com/grpc/grpc-go/issues/7425)) - xds: Implement ADS flow control mechanism ([#​7458](https://togithub.com/grpc/grpc-go/issues/7458)) - See [https://github.com/grpc/grpc/issues/34099](https://togithub.com/grpc/grpc/issues/34099) for context. - balancer/rls: Add metrics for data cache and picker internals ([#​7484](https://togithub.com/grpc/grpc-go/issues/7484), [#​7495](https://togithub.com/grpc/grpc-go/issues/7495)) - xds: LRS load reports now include the `total_issued_requests` field. ([#​7544](https://togithub.com/grpc/grpc-go/issues/7544)) ### Bug Fixes - grpc: Clients now return status code INTERNAL instead of UNIMPLEMENTED when the server uses an unsupported compressor. This is consistent with the [gRPC compression spec](https://togithub.com/grpc/grpc/blob/master/doc/compression.md#compression-method-asymmetry-between-peers). ([#​7461](https://togithub.com/grpc/grpc-go/issues/7461)) - Special Thanks: [@​Gayathri625](https://togithub.com/Gayathri625) - transport: Fix a bug which could result in writes busy looping when the underlying `conn.Write` returns errors ([#​7394](https://togithub.com/grpc/grpc-go/issues/7394)) - Special Thanks: [@​veshij](https://togithub.com/veshij) - client: fix race that could lead to orphaned connections and associated resources. ([#​7390](https://togithub.com/grpc/grpc-go/issues/7390)) - xds: use locality from the connected address for load reporting with pick_first ([#​7378](https://togithub.com/grpc/grpc-go/issues/7378)) - without this fix, if a priority contains multiple localities with pick_first, load was reported for the wrong locality - client: prevent hanging during ClientConn.Close() when the network is unreachable ([#​7540](https://togithub.com/grpc/grpc-go/issues/7540)) ### Performance Improvements - transport: double buffering is avoided when using an http connect proxy and the target server waits for client to send the first message. ([#​7424](https://togithub.com/grpc/grpc-go/issues/7424)) - codec: Implement a new `Codec` which uses buffer recycling for encoded message ([#​7356](https://togithub.com/grpc/grpc-go/issues/7356)) - introduce a `mem` package to facilitate buffer reuse ([#​7432](https://togithub.com/grpc/grpc-go/issues/7432)) - Special Thanks: [@​PapaCharlie](https://togithub.com/PapaCharlie)
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/jaegertracing/jaeger). --------- Signed-off-by: Mend Renovate Signed-off-by: Yuri Shkuro Co-authored-by: Yuri Shkuro Co-authored-by: Yuri Shkuro Signed-off-by: Mahad Zaryab --- cmd/es-index-cleaner/README.md | 4 ++-- pkg/gogocodec/codec.go | 18 ++++++++++-------- pkg/gogocodec/codec_test.go | 34 ++++++++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/cmd/es-index-cleaner/README.md b/cmd/es-index-cleaner/README.md index d6b063c65643..c9b980230757 100644 --- a/cmd/es-index-cleaner/README.md +++ b/cmd/es-index-cleaner/README.md @@ -1,8 +1,8 @@ -# es-index-cleaner +# jaeger-es-index-cleaner It is common to only keep observability data for a limited time. However, Elasticsearch does no support expiring of old data via TTL. -To help with this task, `es-index-cleaner` can be used to purge +To help with this task, `jaeger-es-index-cleaner` can be used to purge old Jaeger indices. For example, to delete indixes older than 14 days: ``` diff --git a/pkg/gogocodec/codec.go b/pkg/gogocodec/codec.go index 96d9d1de81b9..55322baa89c2 100644 --- a/pkg/gogocodec/codec.go +++ b/pkg/gogocodec/codec.go @@ -11,6 +11,7 @@ import ( gogoproto "github.com/gogo/protobuf/proto" "google.golang.org/grpc/encoding" "google.golang.org/grpc/encoding/proto" + "google.golang.org/grpc/mem" ) const ( @@ -18,7 +19,7 @@ const ( jaegerModelPkgPath = "github.com/jaegertracing/jaeger/model" ) -var defaultCodec encoding.Codec +var defaultCodec encoding.CodecV2 // CustomType is an interface that Gogo expects custom types to implement. // https://github.com/gogo/protobuf/blob/master/custom_types.md @@ -34,16 +35,16 @@ type CustomType interface { } func init() { - defaultCodec = encoding.GetCodec(proto.Name) + defaultCodec = encoding.GetCodecV2(proto.Name) defaultCodec.Name() // ensure it's not nil - encoding.RegisterCodec(newCodec()) + encoding.RegisterCodecV2(newCodec()) } // gogoCodec forces the use of gogo proto marshalling/unmarshalling for // Jaeger proto types (package jaeger/gen-proto). type gogoCodec struct{} -var _ encoding.Codec = (*gogoCodec)(nil) +var _ encoding.CodecV2 = (*gogoCodec)(nil) func newCodec() *gogoCodec { return &gogoCodec{} @@ -55,23 +56,24 @@ func (*gogoCodec) Name() string { } // Marshal implements encoding.Codec -func (*gogoCodec) Marshal(v any) ([]byte, error) { +func (*gogoCodec) Marshal(v any) (mem.BufferSlice, error) { t := reflect.TypeOf(v) elem := t.Elem() // use gogo proto only for Jaeger types if useGogo(elem) { - return gogoproto.Marshal(v.(gogoproto.Message)) + bytes, err := gogoproto.Marshal(v.(gogoproto.Message)) + return mem.BufferSlice{mem.SliceBuffer(bytes)}, err } return defaultCodec.Marshal(v) } // Unmarshal implements encoding.Codec -func (*gogoCodec) Unmarshal(data []byte, v any) error { +func (*gogoCodec) Unmarshal(data mem.BufferSlice, v any) error { t := reflect.TypeOf(v) elem := t.Elem() // only for collections // use gogo proto only for Jaeger types if useGogo(elem) { - return gogoproto.Unmarshal(data, v.(gogoproto.Message)) + return gogoproto.Unmarshal(data.Materialize(), v.(gogoproto.Message)) } return defaultCodec.Unmarshal(data, v) } diff --git a/pkg/gogocodec/codec_test.go b/pkg/gogocodec/codec_test.go index daacce2ae991..9349e74c62fa 100644 --- a/pkg/gogocodec/codec_test.go +++ b/pkg/gogocodec/codec_test.go @@ -4,11 +4,14 @@ package gogocodec import ( + "os" "reflect" "testing" + "github.com/gogo/protobuf/jsonpb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc/mem" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -54,14 +57,14 @@ func TestWireCompatibility(t *testing.T) { require.NoError(t, err) var goprotoMessage emptypb.Empty - err = proto.Unmarshal(data, &goprotoMessage) + err = proto.Unmarshal(data.Materialize(), &goprotoMessage) require.NoError(t, err) data2, err := proto.Marshal(&goprotoMessage) require.NoError(t, err) s2 := &model.Span{} - err = c.Unmarshal(data2, s2) + err = c.Unmarshal(mem.BufferSlice{mem.SliceBuffer(data2)}, s2) require.NoError(t, err) assert.Equal(t, s1, s2) } @@ -73,6 +76,33 @@ func TestUseGogo(t *testing.T) { assert.True(t, useGogo(reflect.TypeOf(span))) } +func BenchmarkCodecUnmarshal25Spans(b *testing.B) { + const fileName = "../../model/converter/thrift/jaeger/fixtures/domain_01.json" + jsonFile, err := os.Open(fileName) + require.NoError(b, err, "Failed to open json fixture file %s", fileName) + var trace model.Trace + require.NoError(b, jsonpb.Unmarshal(jsonFile, &trace), fileName) + require.NotEmpty(b, trace.Spans) + spans := make([]*model.Span, 25) + for i := 0; i < len(spans); i++ { + spans[i] = trace.Spans[0] + } + trace.Spans = spans + c := newCodec() + bytes, err := c.Marshal(&trace) + require.NoError(b, err) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var trace model.Trace + err := c.Unmarshal(bytes, &trace) + if err != nil { + b.Fatal(err) + } + } +} + func TestMain(m *testing.M) { testutils.VerifyGoLeaks(m) }