From 10a0efbdac9cce2fb2e9fd8b806810276c315646 Mon Sep 17 00:00:00 2001 From: Injun Song Date: Sat, 24 Feb 2024 20:45:56 +0900 Subject: [PATCH] perf(benchmark): use grpc.SharedBufferPool for grpc.DialOption When a client library parses incoming gRPC messages, it allocates new buffers frequently. This can result in the creation of a large number of heap objects. However, gRPC offers an experimental feature called the shared buffer pool that can be used to address this issue. This pull request (PR) aims to optimize the gRPC receiving path using the shared buffer pool. Various experiments have shown that this optimization leads to a decrease in CPU time of runtime.gcBgMarkWorkers, depending on the workload. Since the shared buffer pool is experimental, it is applied only to the benchmark. Refs: - https://pkg.go.dev/google.golang.org/grpc@v1.62.0/experimental#WithRecvBufferPool - https://pkg.go.dev/google.golang.org/grpc@v1.62.0#SharedBufferPool --- internal/admin/mrmanager/manager_mock.go | 1 + internal/admin/replica_selector_mock.go | 1 + internal/admin/snmanager/manager_mock.go | 1 + internal/admin/snwatcher/snwatcher_mock.go | 1 + internal/admin/stats/repository_mock.go | 1 + internal/benchmark/loader.go | 6 +- internal/reportcommitter/client_mock.go | 1 + .../client/management_client_mock.go | 1 + pkg/mrc/metadata_repository_client_mock.go | 1 + ...adata_repository_management_client_mock.go | 1 + pkg/varlog/admin_mock.go | 1 + pkg/varlog/log_mock.go | 1 + pkg/varlog/log_stream_appender_mock.go | 1 + pkg/varlog/metadata_refresher_mock.go | 1 + pkg/varlog/options.go | 8 +++ pkg/varlog/replicas_retriever_mock.go | 1 + proto/admpb/admin_mock.go | 1 + proto/mrpb/mock/mrpb_mock.go | 1 + proto/snpb/mock/snpb_mock.go | 1 + .../grpc/experimental/experimental.go | 65 +++++++++++++++++++ vendor/modules.txt | 1 + 21 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 vendor/google.golang.org/grpc/experimental/experimental.go diff --git a/internal/admin/mrmanager/manager_mock.go b/internal/admin/mrmanager/manager_mock.go index f65647c5b..a6913b557 100644 --- a/internal/admin/mrmanager/manager_mock.go +++ b/internal/admin/mrmanager/manager_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/admin/mrmanager -package mrmanager -destination manager_mock.go . ClusterMetadataView,MetadataRepositoryManager // + // Package mrmanager is a generated GoMock package. package mrmanager diff --git a/internal/admin/replica_selector_mock.go b/internal/admin/replica_selector_mock.go index ef32d27cb..13931dfb4 100644 --- a/internal/admin/replica_selector_mock.go +++ b/internal/admin/replica_selector_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/admin -package admin -destination replica_selector_mock.go . ReplicaSelector // + // Package admin is a generated GoMock package. package admin diff --git a/internal/admin/snmanager/manager_mock.go b/internal/admin/snmanager/manager_mock.go index 61d9016a9..e5d7dc0ab 100644 --- a/internal/admin/snmanager/manager_mock.go +++ b/internal/admin/snmanager/manager_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/admin/snmanager -package snmanager -destination manager_mock.go . StorageNodeManager // + // Package snmanager is a generated GoMock package. package snmanager diff --git a/internal/admin/snwatcher/snwatcher_mock.go b/internal/admin/snwatcher/snwatcher_mock.go index 6089b3dfa..122d69035 100644 --- a/internal/admin/snwatcher/snwatcher_mock.go +++ b/internal/admin/snwatcher/snwatcher_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/admin/snwatcher -package snwatcher -destination snwatcher_mock.go . EventHandler // + // Package snwatcher is a generated GoMock package. package snwatcher diff --git a/internal/admin/stats/repository_mock.go b/internal/admin/stats/repository_mock.go index 600c566ff..b793a29f5 100644 --- a/internal/admin/stats/repository_mock.go +++ b/internal/admin/stats/repository_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/admin/stats -package stats -destination repository_mock.go . Repository // + // Package stats is a generated GoMock package. package stats diff --git a/internal/benchmark/loader.go b/internal/benchmark/loader.go index 57356081b..4c242a37b 100644 --- a/internal/benchmark/loader.go +++ b/internal/benchmark/loader.go @@ -13,6 +13,8 @@ import ( "go.uber.org/multierr" "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/experimental" "github.com/kakao/varlog/pkg/types" "github.com/kakao/varlog/pkg/varlog" @@ -54,7 +56,9 @@ func NewLoader(cfg loaderConfig) (loader *Loader, err error) { if scli != nil { return scli, nil } - cli, err := varlog.Open(context.TODO(), loader.cid, loader.mraddrs) + cli, err := varlog.Open(context.TODO(), loader.cid, loader.mraddrs, varlog.WithGRPCDialOptions( + experimental.WithRecvBufferPool(grpc.NewSharedBufferPool()), + )) if err != nil { return nil, err } diff --git a/internal/reportcommitter/client_mock.go b/internal/reportcommitter/client_mock.go index fd8a269b5..d1f65e0f5 100644 --- a/internal/reportcommitter/client_mock.go +++ b/internal/reportcommitter/client_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/reportcommitter -package reportcommitter -destination client_mock.go . Client // + // Package reportcommitter is a generated GoMock package. package reportcommitter diff --git a/internal/storagenode/client/management_client_mock.go b/internal/storagenode/client/management_client_mock.go index 67983f4c3..3e4cd2f39 100644 --- a/internal/storagenode/client/management_client_mock.go +++ b/internal/storagenode/client/management_client_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/internal/storagenode/client -package client -destination management_client_mock.go . StorageNodeManagementClient // + // Package client is a generated GoMock package. package client diff --git a/pkg/mrc/metadata_repository_client_mock.go b/pkg/mrc/metadata_repository_client_mock.go index 11fcd8dbb..510b40219 100644 --- a/pkg/mrc/metadata_repository_client_mock.go +++ b/pkg/mrc/metadata_repository_client_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/pkg/mrc -package mrc -destination metadata_repository_client_mock.go . MetadataRepositoryClient // + // Package mrc is a generated GoMock package. package mrc diff --git a/pkg/mrc/metadata_repository_management_client_mock.go b/pkg/mrc/metadata_repository_management_client_mock.go index 52a88237a..a57237078 100644 --- a/pkg/mrc/metadata_repository_management_client_mock.go +++ b/pkg/mrc/metadata_repository_management_client_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/pkg/mrc -package mrc -destination metadata_repository_management_client_mock.go . MetadataRepositoryManagementClient // + // Package mrc is a generated GoMock package. package mrc diff --git a/pkg/varlog/admin_mock.go b/pkg/varlog/admin_mock.go index 8ef5eadff..066dc1a3f 100644 --- a/pkg/varlog/admin_mock.go +++ b/pkg/varlog/admin_mock.go @@ -5,6 +5,7 @@ // // mockgen -package varlog -destination admin_mock.go . Admin // + // Package varlog is a generated GoMock package. package varlog diff --git a/pkg/varlog/log_mock.go b/pkg/varlog/log_mock.go index 80cf0517f..94d29cda2 100644 --- a/pkg/varlog/log_mock.go +++ b/pkg/varlog/log_mock.go @@ -5,6 +5,7 @@ // // mockgen -package varlog -destination log_mock.go . Log // + // Package varlog is a generated GoMock package. package varlog diff --git a/pkg/varlog/log_stream_appender_mock.go b/pkg/varlog/log_stream_appender_mock.go index 1a84f9fab..02d1f9ebc 100644 --- a/pkg/varlog/log_stream_appender_mock.go +++ b/pkg/varlog/log_stream_appender_mock.go @@ -5,6 +5,7 @@ // // mockgen -package varlog -destination log_stream_appender_mock.go . LogStreamAppender // + // Package varlog is a generated GoMock package. package varlog diff --git a/pkg/varlog/metadata_refresher_mock.go b/pkg/varlog/metadata_refresher_mock.go index b1622f74d..ad16614a2 100644 --- a/pkg/varlog/metadata_refresher_mock.go +++ b/pkg/varlog/metadata_refresher_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/pkg/varlog -package varlog -destination metadata_refresher_mock.go . MetadataRefresher // + // Package varlog is a generated GoMock package. package varlog diff --git a/pkg/varlog/options.go b/pkg/varlog/options.go index eeb29b791..12a10bfe1 100644 --- a/pkg/varlog/options.go +++ b/pkg/varlog/options.go @@ -180,6 +180,14 @@ func WithGRPCInitialWindowSize(bytes int32) Option { }) } +// WithGRPCDialOptions sets the grpc dial options. +// See `google.golang.org/grpc.DialOption`. +func WithGRPCDialOptions(grpcDialOptions ...grpc.DialOption) Option { + return newOption(func(opts *options) { + opts.grpcDialOptions = append(opts.grpcDialOptions, grpcDialOptions...) + }) +} + const ( defaultRetryCount = 3 ) diff --git a/pkg/varlog/replicas_retriever_mock.go b/pkg/varlog/replicas_retriever_mock.go index 243d6ed26..498185951 100644 --- a/pkg/varlog/replicas_retriever_mock.go +++ b/pkg/varlog/replicas_retriever_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -self_package github.com/kakao/varlog/pkg/varlog -package varlog -destination replicas_retriever_mock.go . ReplicasRetriever,RenewableReplicasRetriever // + // Package varlog is a generated GoMock package. package varlog diff --git a/proto/admpb/admin_mock.go b/proto/admpb/admin_mock.go index 1fd725dde..b79808aef 100644 --- a/proto/admpb/admin_mock.go +++ b/proto/admpb/admin_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -package admpb -destination admin_mock.go . ClusterManagerClient,ClusterManagerServer // + // Package admpb is a generated GoMock package. package admpb diff --git a/proto/mrpb/mock/mrpb_mock.go b/proto/mrpb/mock/mrpb_mock.go index 8232eab91..105fd38a3 100644 --- a/proto/mrpb/mock/mrpb_mock.go +++ b/proto/mrpb/mock/mrpb_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -package mock -destination mock/mrpb_mock.go . ManagementClient,ManagementServer,MetadataRepositoryServiceClient,MetadataRepositoryServiceServer // + // Package mock is a generated GoMock package. package mock diff --git a/proto/snpb/mock/snpb_mock.go b/proto/snpb/mock/snpb_mock.go index 068b8e9fe..8f8850654 100644 --- a/proto/snpb/mock/snpb_mock.go +++ b/proto/snpb/mock/snpb_mock.go @@ -5,6 +5,7 @@ // // mockgen -build_flags -mod=vendor -package mock -destination mock/snpb_mock.go . ReplicatorClient,ReplicatorServer,Replicator_ReplicateClient,LogIOClient,LogIOServer,LogIO_AppendClient,LogIO_SubscribeClient,LogIO_SubscribeServer,LogStreamReporterClient,LogStreamReporterServer,ManagementClient,ManagementServer // + // Package mock is a generated GoMock package. package mock diff --git a/vendor/google.golang.org/grpc/experimental/experimental.go b/vendor/google.golang.org/grpc/experimental/experimental.go new file mode 100644 index 000000000..de7f13a22 --- /dev/null +++ b/vendor/google.golang.org/grpc/experimental/experimental.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package experimental is a collection of experimental features that might +// have some rough edges to them. Housing experimental features in this package +// results in a user accessing these APIs as `experimental.Foo`, thereby making +// it explicit that the feature is experimental and using them in production +// code is at their own risk. +// +// All APIs in this package are experimental. +package experimental + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/internal" +) + +// WithRecvBufferPool returns a grpc.DialOption that configures the use of +// bufferPool for parsing incoming messages on a grpc.ClientConn. Depending on +// the application's workload, this could result in reduced memory allocation. +// +// If you are unsure about how to implement a memory pool but want to utilize +// one, begin with grpc.NewSharedBufferPool. +// +// Note: The shared buffer pool feature will not be active if any of the +// following options are used: WithStatsHandler, EnableTracing, or binary +// logging. In such cases, the shared buffer pool will be ignored. +// +// Note: It is not recommended to use the shared buffer pool when compression is +// enabled. +func WithRecvBufferPool(bufferPool grpc.SharedBufferPool) grpc.DialOption { + return internal.WithRecvBufferPool.(func(grpc.SharedBufferPool) grpc.DialOption)(bufferPool) +} + +// RecvBufferPool returns a grpc.ServerOption that configures the server to use +// the provided shared buffer pool for parsing incoming messages. Depending on +// the application's workload, this could result in reduced memory allocation. +// +// If you are unsure about how to implement a memory pool but want to utilize +// one, begin with grpc.NewSharedBufferPool. +// +// Note: The shared buffer pool feature will not be active if any of the +// following options are used: StatsHandler, EnableTracing, or binary logging. +// In such cases, the shared buffer pool will be ignored. +// +// Note: It is not recommended to use the shared buffer pool when compression is +// enabled. +func RecvBufferPool(bufferPool grpc.SharedBufferPool) grpc.ServerOption { + return internal.RecvBufferPool.(func(grpc.SharedBufferPool) grpc.ServerOption)(bufferPool) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 267d8b5de..0aa088309 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -557,6 +557,7 @@ google.golang.org/grpc/credentials/insecure google.golang.org/grpc/encoding google.golang.org/grpc/encoding/gzip google.golang.org/grpc/encoding/proto +google.golang.org/grpc/experimental google.golang.org/grpc/grpclog google.golang.org/grpc/health google.golang.org/grpc/health/grpc_health_v1