diff --git a/docs/arenastring.en.md b/docs/arenastring.en.md index e8d9d705..f49d2111 100644 --- a/docs/arenastring.en.md +++ b/docs/arenastring.en.md @@ -41,3 +41,5 @@ common --registry=https://baidu.github.io/babylon/registry # in MODULE.bazel bazel_dep(name = 'protobuf', version = '28.3.arenastring') ``` + +Here is an example of using this patch with [brpc](https://github.com/apache/brpc): [use-arena-with-brpc](../example/use-arena-with-brpc), along with some performance demonstrations. diff --git a/docs/arenastring.zh-cn.md b/docs/arenastring.zh-cn.md index a38640b6..7c7e96b7 100644 --- a/docs/arenastring.zh-cn.md +++ b/docs/arenastring.zh-cn.md @@ -33,3 +33,5 @@ common --registry=https://baidu.github.io/babylon/registry # in MODULE.bazel bazel_dep(name = 'protobuf', version = '28.3.arenastring') ``` + +这里可以找到一个结合[brpc](https://github.com/apache/brpc)使用补丁的例子[use-arena-with-brpc](../example/use-arena-with-brpc)以及一些性能演示 diff --git a/example/anyflow/MODULE.bazel b/example/anyflow/MODULE.bazel index c95c29be..e9bf56bb 100644 --- a/example/anyflow/MODULE.bazel +++ b/example/anyflow/MODULE.bazel @@ -1 +1 @@ -bazel_dep(name = 'babylon', version = '1.4.1') +bazel_dep(name = 'babylon', version = '1.4.2') diff --git a/example/depend-use-bzlmod/MODULE.bazel b/example/depend-use-bzlmod/MODULE.bazel index c95c29be..e9bf56bb 100644 --- a/example/depend-use-bzlmod/MODULE.bazel +++ b/example/depend-use-bzlmod/MODULE.bazel @@ -1 +1 @@ -bazel_dep(name = 'babylon', version = '1.4.1') +bazel_dep(name = 'babylon', version = '1.4.2') diff --git a/example/depend-use-bzlmod/README.md b/example/depend-use-bzlmod/README.md index 15b686a1..44d28549 100644 --- a/example/depend-use-bzlmod/README.md +++ b/example/depend-use-bzlmod/README.md @@ -5,7 +5,7 @@ - 增加依赖项 ``` # in MODULE.bazel -bazel_dep(name = 'babylon', version = '1.4.1') +bazel_dep(name = 'babylon', version = '1.4.2') ``` - 添加依赖的子模块到编译目标，全部可用子模块可以参照[模块功能文档](../../docs/README.zh-cn.md#模块功能文档)，或者[BUILD](../../BUILD)文件，也可以直接添加All in One依赖目标`@babylon` diff --git a/example/depend-use-cmake-fetch/CMakeLists.txt b/example/depend-use-cmake-fetch/CMakeLists.txt index 616dd86f..a3411ecd 100644 --- a/example/depend-use-cmake-fetch/CMakeLists.txt +++ b/example/depend-use-cmake-fetch/CMakeLists.txt @@ -5,8 +5,8 @@ set(BUILD_DEPS ON) include(FetchContent) FetchContent_Declare( babylon - URL "https://github.com/baidu/babylon/releases/download/v1.4.1/v1.4.1.tar.gz" - URL_HASH SHA256=930e8d24822a472466e8b616011a57c37021b02486ad19ee7b62c12bfef923b8 + URL "https://github.com/baidu/babylon/releases/download/v1.4.2/v1.4.2.tar.gz" + URL_HASH SHA256=d60ee9cd86a777137bf021c8861e97438a69cc857659d5eb39af9e8464434cf1 ) FetchContent_MakeAvailable(babylon) diff --git a/example/depend-use-cmake-fetch/README.md b/example/depend-use-cmake-fetch/README.md index 4533d870..2864fd49 100644 --- a/example/depend-use-cmake-fetch/README.md +++ b/example/depend-use-cmake-fetch/README.md @@ -10,8 +10,8 @@ set(BUILD_DEPS ON) include(FetchContent) FetchContent_Declare( babylon - URL "https://github.com/baidu/babylon/releases/download/v1.4.1/v1.4.1.tar.gz" - URL_HASH SHA256=930e8d24822a472466e8b616011a57c37021b02486ad19ee7b62c12bfef923b8 + URL "https://github.com/baidu/babylon/releases/download/v1.4.2/v1.4.2.tar.gz" + URL_HASH SHA256=d60ee9cd86a777137bf021c8861e97438a69cc857659d5eb39af9e8464434cf1 ) FetchContent_MakeAvailable(babylon) ``` diff --git a/example/depend-use-cmake-find/build.sh b/example/depend-use-cmake-find/build.sh index 51500f25..1900d254 100755 --- a/example/depend-use-cmake-find/build.sh +++ b/example/depend-use-cmake-find/build.sh @@ -1,9 +1,9 @@ #!/bin/sh set -ex -URL=https://github.com/baidu/babylon/releases/download/v1.4.1/v1.4.1.tar.gz +URL=https://github.com/baidu/babylon/releases/download/v1.4.2/v1.4.2.tar.gz NAME=babylon-1.4.1 -SHA256=930e8d24822a472466e8b616011a57c37021b02486ad19ee7b62c12bfef923b8 +SHA256=d60ee9cd86a777137bf021c8861e97438a69cc857659d5eb39af9e8464434cf1 if ! echo "$SHA256 $NAME.tar.gz" | sha256sum -c; then wget $URL --continue -O $NAME.tar.gz fi diff --git a/example/depend-use-cmake-subdir/build.sh b/example/depend-use-cmake-subdir/build.sh index a3d6d8eb..badc6919 100755 --- a/example/depend-use-cmake-subdir/build.sh +++ b/example/depend-use-cmake-subdir/build.sh @@ -1,9 +1,9 @@ #!/bin/sh set -ex -URL=https://github.com/baidu/babylon/releases/download/v1.4.1/v1.4.1.tar.gz -NAME=babylon-1.4.1 -SHA256=930e8d24822a472466e8b616011a57c37021b02486ad19ee7b62c12bfef923b8 +URL=https://github.com/baidu/babylon/releases/download/v1.4.2/v1.4.2.tar.gz +NAME=babylon-1.4.2 +SHA256=d60ee9cd86a777137bf021c8861e97438a69cc857659d5eb39af9e8464434cf1 if ! echo "$SHA256 $NAME.tar.gz" | sha256sum -c; then wget $URL --continue -O $NAME.tar.gz fi diff --git a/example/depend-use-workspace/README.md b/example/depend-use-workspace/README.md index b5a545a2..2ba7f9ba 100644 --- a/example/depend-use-workspace/README.md +++ b/example/depend-use-workspace/README.md @@ -7,9 +7,9 @@ # in WORKSPACE http_archive( name = 'com_baidu_babylon', - urls = ['https://github.com/baidu/babylon/releases/download/v1.4.1/v1.4.1.tar.gz'], - strip_prefix = 'babylon-1.4.1', - sha256 = '930e8d24822a472466e8b616011a57c37021b02486ad19ee7b62c12bfef923b8', + urls = ['https://github.com/baidu/babylon/releases/download/v1.4.2/v1.4.2.tar.gz'], + strip_prefix = 'babylon-1.4.2', + sha256 = 'd60ee9cd86a777137bf021c8861e97438a69cc857659d5eb39af9e8464434cf1', ) ``` diff --git a/example/depend-use-workspace/WORKSPACE b/example/depend-use-workspace/WORKSPACE index 0508ef53..25559706 100644 --- a/example/depend-use-workspace/WORKSPACE +++ b/example/depend-use-workspace/WORKSPACE @@ -6,9 +6,9 @@ load('@bazel_tools//tools/build_defs/repo:http.bzl', 'http_archive') # babylon http_archive( name = 'com_baidu_babylon', - urls = ['https://github.com/baidu/babylon/releases/download/v1.4.1/v1.4.1.tar.gz'], - strip_prefix = 'babylon-1.4.1', - sha256 = '930e8d24822a472466e8b616011a57c37021b02486ad19ee7b62c12bfef923b8', + urls = ['https://github.com/baidu/babylon/releases/download/v1.4.2/v1.4.2.tar.gz'], + strip_prefix = 'babylon-1.4.2', + sha256 = 'd60ee9cd86a777137bf021c8861e97438a69cc857659d5eb39af9e8464434cf1', ) ################################################################################ diff --git a/example/use-anyflow-with-brpc/MODULE.bazel b/example/use-anyflow-with-brpc/MODULE.bazel index e4d95e8c..d2386427 100644 --- a/example/use-anyflow-with-brpc/MODULE.bazel +++ b/example/use-anyflow-with-brpc/MODULE.bazel @@ -1,3 +1,3 @@ -bazel_dep(name = 'babylon', version = '1.4.1') -bazel_dep(name = 'brpc', version = '1.10.0.bcr.1') +bazel_dep(name = 'babylon', version = '1.4.2') +bazel_dep(name = 'brpc', version = '1.11.0') bazel_dep(name = 'yaml-cpp', version = '0.8.0') diff --git a/example/use-arena-with-brpc/.bazelversion b/example/use-arena-with-brpc/.bazelversion index eab246c0..ba7f754d 100644 --- a/example/use-arena-with-brpc/.bazelversion +++ b/example/use-arena-with-brpc/.bazelversion @@ -1 +1 @@ -7.3.2 +7.4.0 diff --git a/example/use-arena-with-brpc/BUILD b/example/use-arena-with-brpc/BUILD index ccb3a719..b405d859 100644 --- a/example/use-arena-with-brpc/BUILD +++ b/example/use-arena-with-brpc/BUILD @@ -1,11 +1,10 @@ cc_library( - name = 'reusable_rpc_protocol', - srcs = ['reusable_rpc_protocol.cpp', 'reusable_rpc_protocol.trick.cpp'], - hdrs = ['reusable_rpc_protocol.h'], + name = 'swiss_message_factory', + srcs = ['swiss_message_factory.cpp'], + hdrs = ['swiss_message_factory.h'], deps = [ '@brpc//:brpc', '@babylon//:concurrent_object_pool', - '@babylon//:concurrent_transient_hash_table', '@babylon//:reusable', ], ) @@ -35,7 +34,18 @@ cc_binary( srcs = ['server.cpp'], deps = [ ':cc_echo_proto', - ':reusable_rpc_protocol', + '@brpc//:brpc', + '@tcmalloc//tcmalloc', + ], +) + +cc_binary( + name = 'server_babylon', + srcs = ['server.cpp'], + copts = ['-DWITH_BABYLON=1'], + deps = [ + ':cc_echo_proto', + ':swiss_message_factory', '@brpc//:brpc', '@tcmalloc//tcmalloc', ], diff --git a/example/use-arena-with-brpc/MODULE.bazel b/example/use-arena-with-brpc/MODULE.bazel index 91805e12..7c162673 100644 --- a/example/use-arena-with-brpc/MODULE.bazel +++ b/example/use-arena-with-brpc/MODULE.bazel @@ -1,4 +1,18 @@ -bazel_dep(name = 'babylon', version = '1.4.1') -bazel_dep(name = 'brpc', version = '1.9.0.bcr.1') +bazel_dep(name = 'babylon', version = '1.4.2') +bazel_dep(name = 'brpc', version = '1.11.0') bazel_dep(name = 'tcmalloc', version = '0.0.0-20240411-5ed309d') -single_version_override(module_name = 'protobuf', version = '25.3.arenastring') + +single_version_override(module_name = 'rules_fuzzing', version = '0.5.1') +#single_version_override(module_name = 'protobuf', version = '28.3') +single_version_override(module_name = 'protobuf', version = '28.3.arenastring') + +archive_override(module_name = 'brpc', + urls = ['https://github.com/apache/brpc/archive/30a56dd153010c0be88d7c197712e5b95843ff9c.zip'], + integrity = 'sha256-94aE0H4NZo72DSB0wtq6YlMZc9fOpGAlK4QHgG0FvXg=', + strip_prefix = 'brpc-30a56dd153010c0be88d7c197712e5b95843ff9c', +) + +local_path_override( + module_name = 'babylon', + path = '../..', +) diff --git a/example/use-arena-with-brpc/README.en.md b/example/use-arena-with-brpc/README.en.md new file mode 100644 index 00000000..db28016a --- /dev/null +++ b/example/use-arena-with-brpc/README.en.md @@ -0,0 +1,30 @@ +# Use Arena for brpc + +Before invoking a user's service, [brpc](https://github.com/apache/brpc) needs to construct instances of `Request` and `Response` internally, as well as perform corresponding serialization and deserialization operations. By default, it uses dynamic heap memory allocation. For complex structures, the allocation and deallocation of memory, along with the construction and destruction of `Message` structures, can lead to noticeable overhead. + +Since version 3.x, [Protobuf](https://github.com/protocolbuffers/protobuf) has introduced [Arena](https://protobuf.dev/reference/cpp/arenas) allocation, which enables aggregated allocation and deallocation for complex structures. More recent versions of [brpc](https://github.com/apache/brpc) also support the [Protobuf arena](https://github.com/apache/brpc/blob/master/docs/cn/server.md#protobuf-arena) component. Based on this, further acceleration for `string` members can be achieved by applying [arenastring](../../docs/arenastring.zh-cn.md). + +In addition to using native [Arena](https://protobuf.dev/reference/cpp/arenas), you can also employ [babylon::SwissMemoryResource](../../docs/reusable/memory_resource.zh-cn.md#swissmemoryresource) for memory pool acceleration. [babylon::SwissMemoryResource](../../docs/reusable/memory_resource.zh-cn.md#swissmemoryresource) enables further flexibility through a customizable fixed-size paging reuse mechanism. + +## Performance Demonstration + +CPU: AMD EPYC 7W83 64-Core Processor, taskset 0-3 core + +QPS: 800 + +- Default (mode: 0) + - latency_percentiles: "[2213,2523,3232,3670]" + - process_cpu_usage : 1.172 + - process_memory_resident : 44978722 +- Arena (mode: 1) + - latency_percentiles: "[1318,1490,1794,1984]" + - process_cpu_usage : 0.702 + - process_memory_resident : 41421824 +- Arena & ArenaString (mode: 1, arenastring patch) + - latency_percentiles: "[1055,1196,1416,1583]" + - process_cpu_usage : 0.572 + - process_memory_resident : 39732770 +- SwissMemoryResource & ArenaString (mode: 2, arenastring patch) + - latency_percentiles: "[1006,1139,1341,1478]" + - process_cpu_usage : 0.551 + - process_memory_resident : 44763136 diff --git a/example/use-arena-with-brpc/README.md b/example/use-arena-with-brpc/README.md deleted file mode 100644 index 7ab9087c..00000000 --- a/example/use-arena-with-brpc/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# Use arena for brpc - -brpc在调用用户的service前，需要在内部先完成Request和Response的实例构建和以及前后对应的正反序列化。对应的代码实现在相应的Protocol中，默认的方式实例采用动态堆内存分配模式创建，对于比较复杂的结构，内存分配释放和Message结构的构建和析构可能也会带来可见的开销。 - -利用babylon::SwissMemoryResource可以将堆内存分配该用Arena机制分配到内存池上，降低内存分配的成本且提升局部性。进一步使用babylon::ReusableManager可以在保持内存池聚集分配的局部性的同时，进一步通过尽可能复用Message结构降低构建和析构的开销。 - -下面实现了一个集成了对应功能的brpc::Protocol，演示了响应功能的使用方式，并配套对应的例子来演示性能对比。相应的Protocol实际也在baidu内部广泛使用，预期可以支持在生产环境直接使用。 - -## 示例构成 - -- `:reusable_rpc_protocol`: 独立的brpc::Protocol实现，集成了内存复用和实例复用的功能 - - `reusable_rpc_protocol.h`&`reusable_rpc_protocol.cpp`: 独立逻辑，和对应brpc版本无关 - - `reusable_rpc_protocol.trick.cpp`: 拷贝自`src/brpc/policy/baidu_rpc_protocol.cpp`并进行简要修改 -- `:client`&`:server`: 模拟比较复杂的Message演示性能对比 - -## 使用手册 - -``` -#include "reusable_rpc_protocol.h" - -// 向brpc注册新的protocol，默认使用 -// protocol type = 72 -// protocol name = "baidu_std_reuse" -if (0 != ::babylon::ReusableRPCProtocol::register_protocol()) { - // 注册protocol失败 -} -// 返回失败很可能因为type冲突，可以更换type和name -if (0 != ::babylon::ReusableRPCProtocol::register_protocol(type, name)) { - // 注册protocol失败 -} - -// ReusableRPCProtocol协议和baidu_std相同，注册后，默认依然会走baidu_std -// 需要通过显式在option中指定来启用 -::baidu::rpc::ServerOptions options; -options.enabled_protocols = "baidu_std_reuse"; - -// 下面正常注册服务，启动服务器即可 -class SomeServiceImpl : public SomeService { - public: - virtual void some_method(::google::protobuf::RpcController* controller, - const SomeRequest* request, - SomeResponse* response, - ::google::protobuf::Closure* done) { - ... // 正常进行业务处理，对应的request和response已经改用内存池或者实例复用托管了 - } -}; - -// 影响运行时的flag -// --babylon_rpc_full_reuse，是否启用实例重用，默认false -// --babylon_rpc_closure_cache_num，内存池和ReusableManager实例本身也会通过对象池复用，设置对象池大小 -// --babylon_rpc_page_size，内存池单页大小，超过单页大小的申请会直接改为动态申请 -// --babylon_rpc_page_cache_num，内存池页本身通过对象池复用，设置对象池大小 -``` - -## 性能演示 - -CPU: AMD EPYC 7W83 64-Core Processor, taskset 0-3 core - -QPS: 750 - -- 原始模式 - - latency_percentiles: "[1923,2222,2944,3447]" - - process_cpu_usage : 1.489 - - process_memory_resident : 59244544 -- `--use_arena`模式 - - latency_percentiles: "[1378,1607,2263,2716]" - - process_cpu_usage : 0.695 - - process_memory_resident : 54255616 -- `--use_arena`&`--babylon_rpc_full_reuse`模式 - - latency_percentiles: "[1096,1256,1612,1938]" - - process_cpu_usage : 0.612 - - process_memory_resident : 101576704 diff --git a/example/use-arena-with-brpc/README.md b/example/use-arena-with-brpc/README.md new file mode 120000 index 00000000..b636b478 --- /dev/null +++ b/example/use-arena-with-brpc/README.md @@ -0,0 +1 @@ +README.zh-cn.md \ No newline at end of file diff --git a/example/use-arena-with-brpc/README.zh-cn.md b/example/use-arena-with-brpc/README.zh-cn.md new file mode 100644 index 00000000..1de7f748 --- /dev/null +++ b/example/use-arena-with-brpc/README.zh-cn.md @@ -0,0 +1,30 @@ +# Use arena for brpc + +[brpc](https://github.com/apache/brpc)在调用用户的service前，需要在内部先完成Request和Response的实例构建，并在service前后执行对应的正反序列化。默认采用动态堆内存分配模式创建，对于比较复杂的结构，内存分配释放和Message结构的构建和析构可能也会带来可见的开销。 + +[Protobuf](https://github.com/protocolbuffers/protobuf)在3.x之后增加了[Arena](https://protobuf.dev/reference/cpp/arenas)分配功能，针对复杂结构提供了聚集分配和释放能力。较新版本的[brpc](https://github.com/apache/brpc)也提供了[Protobuf arena](https://github.com/apache/brpc/blob/master/docs/cn/server.md#protobuf-arena)组件进行支持。在这些基础上，通过应用[arenastirng](../../docs/arenastring.zh-cn.md)可以针对string成员实现进一步加速。 + +除了使用原生的[Arena](https://protobuf.dev/reference/cpp/arenas)，也可以使用[babylon::SwissMemoryResource](../../docs/reusable/memory_resource.zh-cn.md#swissmemoryresource)实现内存池加速。[babylon::SwissMemoryResource](../../docs/reusable/memory_resource.zh-cn.md#swissmemoryresource)通过可定制的定长分页重用机制，可以进一步提升灵活性。 + +## 性能演示 + +CPU: AMD EPYC 7W83 64-Core Processor, taskset 0-3 core + +QPS: 800 + +- Default (mode: 0) + - latency_percentiles: "[2213,2523,3232,3670]" + - process_cpu_usage : 1.172 + - process_memory_resident : 44978722 +- Arena (mode: 1) + - latency_percentiles: "[1318,1490,1794,1984]" + - process_cpu_usage : 0.702 + - process_memory_resident : 41421824 +- Arena & ArenaString (mode: 1, arenastring patch) + - latency_percentiles: "[1055,1196,1416,1583]" + - process_cpu_usage : 0.572 + - process_memory_resident : 39732770 +- SwissMemoryResource & ArenaString (mode: 2, arenastring patch) + - latency_percentiles: "[1006,1139,1341,1478]" + - process_cpu_usage : 0.551 + - process_memory_resident : 44763136 diff --git a/example/use-arena-with-brpc/build.sh b/example/use-arena-with-brpc/build.sh index f22edc3b..72e06778 100755 --- a/example/use-arena-with-brpc/build.sh +++ b/example/use-arena-with-brpc/build.sh @@ -1,4 +1,4 @@ #!/bin/sh set -ex -bazel build --registry=https://bcr.bazel.build --registry=https://baidu.github.io/babylon/registry --compilation_mode=opt --cxxopt=-std=c++17 client server +bazel build --registry=file:///home/oathdruid/src/babylon/registry --registry=https://bcr.bazel.build --registry=https://baidu.github.io/babylon/registry --compilation_mode=opt --cxxopt=-std=c++17 client server server_babylon diff --git a/example/use-arena-with-brpc/client.cpp b/example/use-arena-with-brpc/client.cpp index 454f7737..6d5e10ac 100644 --- a/example/use-arena-with-brpc/client.cpp +++ b/example/use-arena-with-brpc/client.cpp @@ -10,7 +10,7 @@ DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short"); DEFINE_string(server, "0.0.0.0:8000", "IP Address of server"); DEFINE_int32(timeout_ms, 500, "RPC timeout in milliseconds"); -DEFINE_uint64(qps, 100, ""); +DEFINE_uint64(qps, 750, ""); DEFINE_uint64(payload_scale, 10, ""); diff --git a/example/use-arena-with-brpc/reusable_rpc_protocol.cpp b/example/use-arena-with-brpc/reusable_rpc_protocol.cpp deleted file mode 100644 index 31f16697..00000000 --- a/example/use-arena-with-brpc/reusable_rpc_protocol.cpp +++ /dev/null @@ -1,194 +0,0 @@ -#include "reusable_rpc_protocol.h" - -#include "babylon/reusable/allocator.h" - -DEFINE_uint64( - babylon_rpc_closure_cache_num, 128, - "max idle closure and babylon::SwissMemoryResource num cached for reuse"); -DEFINE_uint64(babylon_rpc_page_size, 128UL << 10, - "each memory block size for allocate request and response"); -DEFINE_uint64(babylon_rpc_page_cache_num, 1024, - "max idle page num cached for reuse"); - -DEFINE_bool(babylon_rpc_full_reuse, false, - "reuse instance instead of only memory"); - -BABYLON_NAMESPACE_BEGIN - -ReusableRPCProtocol::Closure* ReusableRPCProtocol::create( - const MethodProperty* property) noexcept { - ObjectPool* pool; - if (FLAGS_babylon_rpc_full_reuse) { - pool = &closure_pool(property->status); - } else { - pool = &closure_pool(); - } - - auto closure = pool->pop().release(); - closure->set_pool(*pool); - closure->prepare(property); - return closure; -} - -int ReusableRPCProtocol::register_protocol() noexcept { - return register_protocol(72, "baidu_std_reuse"); -} - -int ReusableRPCProtocol::register_protocol(int type, StringView name) noexcept { - static ::std::string protocol_name; - - static ::std::atomic_flag regisitered; - if (regisitered.test_and_set()) { - return -1; - } - - auto protocol_type = ::brpc::ProtocolType(type); - protocol_name = name; - return ::brpc::RegisterProtocol(protocol_type, - ReusableRPCProtocol(protocol_name.c_str())); -} - -PageAllocator& ReusableRPCProtocol::page_allocator() noexcept { - static struct S { - S() noexcept { - new_delete_page_allocator.set_page_size(FLAGS_babylon_rpc_page_size); - cached_page_allocator.set_upstream(new_delete_page_allocator); - cached_page_allocator.set_free_page_capacity( - FLAGS_babylon_rpc_page_cache_num); - } - ::babylon::NewDeletePageAllocator new_delete_page_allocator; - ::babylon::CachedPageAllocator cached_page_allocator; - } singleton; - - struct SS { - static size_t free_page_num(void*) noexcept { - return singleton.cached_page_allocator.free_page_num(); - } - static ::bvar::Stat hit_stat(void*) noexcept { - auto summary = singleton.cached_page_allocator.cache_hit_summary(); - ::bvar::Stat result; - result.sum = summary.sum; - result.num = summary.num; - return result; - } - }; - static ::bvar::PassiveStatus free_page_num( - "babylon_reusable_rpc_free_page_num", SS::free_page_num, nullptr); - static ::bvar::PassiveStatus<::bvar::Stat> hit_stat(SS::hit_stat, nullptr); - static ::bvar::Window<::bvar::PassiveStatus<::bvar::Stat>, - ::bvar::SERIES_IN_SECOND> - window_hit_summary("babylon_reusable_rpc_page_cache_hit_ratio", &hit_stat, - -1); - - return singleton.cached_page_allocator; -} - -ObjectPool& -ReusableRPCProtocol::closure_pool() noexcept { - static struct S { - S() noexcept { - pool.set_creator([&]() { - auto closure = new Closure; - closure->set_page_allocator(page_allocator()); - return ::std::unique_ptr {closure}; - }); - pool.reserve_and_clear(FLAGS_babylon_rpc_closure_cache_num); - } - ObjectPool pool; - } singleton; - return singleton.pool; -} - -ObjectPool& ReusableRPCProtocol::closure_pool( - MethodStatus* method_status) noexcept { - static ConcurrentTransientHashMap> pools { - 32}; - auto iter = pools.find(method_status); - if (iter != pools.end()) { - return iter->second; - } - - ObjectPool pool; - pool.set_creator([&]() { - auto closure = new Closure; - closure->set_page_allocator(page_allocator()); - return ::std::unique_ptr {closure}; - }); - pool.reserve_and_clear(FLAGS_babylon_rpc_closure_cache_num); - - auto result = pools.emplace(method_status, ::std::move(pool)); - return result.first->second; -} - -void ReusableRPCProtocol::Closure::set_page_allocator( - PageAllocator& page_allocator) noexcept { - resource().set_page_allocator(page_allocator); -} - -void ReusableRPCProtocol::Closure::set_pool( - ObjectPool& pool) noexcept { - _pool = &pool; -} - -void ReusableRPCProtocol::Closure::prepare( - const ::brpc::Server::MethodProperty* property) noexcept { - if (FLAGS_babylon_rpc_full_reuse) { - if (_method_status == nullptr) { - auto service = property->service; - auto method = property->method; - auto arena = &static_cast(resource()); - _request_accessor = - _manager.create_object([&](SwissMemoryResource&) { - return service->GetRequestPrototype(method).New(arena); - }); - _response_accessor = - _manager.create_object([&](SwissMemoryResource&) { - return service->GetResponsePrototype(method).New(arena); - }); - _method_status = property->status; - } - _request = _request_accessor.get(); - _response = _response_accessor.get(); - } else { - auto service = property->service; - auto method = property->method; - auto arena = &static_cast(resource()); - _request = service->GetRequestPrototype(method).New(arena); - _response = service->GetResponsePrototype(method).New(arena); - _method_status = property->status; - } -} - -void ReusableRPCProtocol::Closure::set_correlation_id( - int64_t correlation_id) noexcept { - _correlation_id = correlation_id; -} - -void ReusableRPCProtocol::Closure::set_received_us( - int64_t received_us) noexcept { - _received_us = received_us; -} - -void ReusableRPCProtocol::Closure::set_server( - const ::brpc::Server* server) noexcept { - _server = server; -} - -void ReusableRPCProtocol::Closure::set_controller( - Controller* controller) noexcept { - _controller = controller; -} - -::google::protobuf::Message* ReusableRPCProtocol::Closure::request() noexcept { - return _request; -} - -::google::protobuf::Message* ReusableRPCProtocol::Closure::response() noexcept { - return _response; -} - -SwissMemoryResource& ReusableRPCProtocol::Closure::resource() noexcept { - return _manager.resource(); -} - -BABYLON_NAMESPACE_END diff --git a/example/use-arena-with-brpc/reusable_rpc_protocol.h b/example/use-arena-with-brpc/reusable_rpc_protocol.h deleted file mode 100644 index c19aed67..00000000 --- a/example/use-arena-with-brpc/reusable_rpc_protocol.h +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once - -#include "babylon/concurrent/object_pool.h" -#include "babylon/concurrent/transient_hash_table.h" -#include "babylon/reusable/manager.h" - -#include "brpc/policy/baidu_rpc_protocol.h" -#include "brpc/server.h" - -BABYLON_NAMESPACE_BEGIN - -// 特化支持server端的protocol -// 逻辑上等同于baidu_std，实际代码也是拷贝而来 -// 主要针对reqeust和response的重复利用，以及生命周期做了扩展支持 -class ReusableRPCProtocol : public ::brpc::Protocol { - private: - using Arena = ::google::protobuf::Arena; - using Message = ::google::protobuf::Message; - using MethodStatus = ::brpc::MethodStatus; - using Controller = ::brpc::Controller; - using MethodProperty = ::brpc::Server::MethodProperty; - - public: - // 传入service代码的回调闭包 - // 内部持有一次请求处理使用的上下文信息 - // Run内部含【回包】和【释放资源】两部分 - // 分别对应send_response和release，即Run = send_response + release - // 通过拆分出来支持两阶段调用，在高级场景可以先回包 - // 之后利用reqeust继续后台处理（异步打印、后台通信等） - class Closure : public ::google::protobuf::Closure { - public: - inline Closure() = default; - Closure(Closure&&) = delete; - Closure(const Closure&) = delete; - Closure& operator=(Closure&&) = delete; - Closure& operator=(const Closure&) = delete; - virtual ~Closure() noexcept = default; - - void set_page_allocator(PageAllocator& page_allocator) noexcept; - - void set_pool(ObjectPool& pool) noexcept; - void prepare(const ::brpc::Server::MethodProperty* property) noexcept; - void set_correlation_id(int64_t correlation_id) noexcept; - void set_received_us(int64_t received_us) noexcept; - void set_server(const ::brpc::Server* server) noexcept; - void set_controller(Controller* controller) noexcept; - - Message* request() noexcept; - Message* response() noexcept; - - virtual void Run() noexcept override; - - private: - SwissMemoryResource& resource() noexcept; - - // 实际的生命周期持有者 - ObjectPool* _pool {nullptr}; - // 对应的RPC method - MethodStatus* _method_status {nullptr}; - // 内存分配器 - SwissManager _manager; - - // 请求级信息 - int64_t _correlation_id {0}; - int64_t _received_us {0}; - const ::brpc::Server* _server {nullptr}; - Controller* _controller {nullptr}; - - // 动态分配的实例 - Message* _request {nullptr}; - Message* _response {nullptr}; - - // 复用模式的持有器 - ReusableAccessor _request_accessor; - ReusableAccessor _response_accessor; - }; - - static int register_protocol() noexcept; - static int register_protocol(int type, StringView name) noexcept; - - static Closure* create(const MethodProperty* property) noexcept; - - private: - static ObjectPool& closure_pool() noexcept; - static PageAllocator& page_allocator() noexcept; - static ObjectPool& closure_pool( - MethodStatus* method_status) noexcept; - - ReusableRPCProtocol(const char* name) noexcept; -}; - -BABYLON_NAMESPACE_END diff --git a/example/use-arena-with-brpc/reusable_rpc_protocol.trick.cpp b/example/use-arena-with-brpc/reusable_rpc_protocol.trick.cpp deleted file mode 100644 index 33aecb4a..00000000 --- a/example/use-arena-with-brpc/reusable_rpc_protocol.trick.cpp +++ /dev/null @@ -1,544 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//////////////////////////////////////////////////////////////////////////////// -// 以下主体拷贝自 -// https://github.com/apache/brpc/blob/1.9.0/src/brpc/policy/baidu_rpc_protocol.cpp -// 修改其中正反序列化相关部分 -// clang-format off - -#include "reusable_rpc_protocol.h" - -#include // MethodDescriptor -#include // Message -#include -#include -#include "butil/logging.h" // LOG() -#include "butil/time.h" -#include "butil/iobuf.h" // butil::IOBuf -#include "butil/raw_pack.h" // RawPacker RawUnpacker -#include "brpc/controller.h" // Controller -#include "brpc/socket.h" // Socket -#include "brpc/server.h" // Server -#include "brpc/span.h" -#include "brpc/compress.h" // ParseFromCompressedData -#include "brpc/stream_impl.h" -#include "brpc/rpc_dump.h" // SampledRequest -#include "brpc/policy/baidu_rpc_meta.pb.h" // RpcRequestMeta -#include "brpc/policy/baidu_rpc_protocol.h" -#include "brpc/policy/most_common_message.h" -#include "brpc/policy/streaming_rpc_protocol.h" -#include "brpc/details/usercode_backup_pool.h" -#include "brpc/details/controller_private_accessor.h" -#include "brpc/details/server_private_accessor.h" - -extern "C" { -void bthread_assign_data(void* data); -} - - -namespace brpc { -namespace policy { - -// Notes: -// 1. 12-byte header [PRPC][body_size][meta_size] -// 2. body_size and meta_size are in network byte order -// 3. Use service->full_name() + method_name to specify the method to call -// 4. `attachment_size' is set iff request/response has attachment -// 5. Not supported: chunk_info - -// Pack header into `buf' -inline void PackRpcHeader(char* rpc_header, uint32_t meta_size, int payload_size) { - uint32_t* dummy = (uint32_t*)rpc_header; // suppress strict-alias warning - *dummy = *(uint32_t*)"PRPC"; - butil::RawPacker(rpc_header + 4) - .pack32(meta_size + payload_size) - .pack32(meta_size); -} - -static void SerializeRpcHeaderAndMeta( - butil::IOBuf* out, const RpcMeta& meta, int payload_size) { - const uint32_t meta_size = GetProtobufByteSize(meta); - if (meta_size <= 244) { // most common cases - char header_and_meta[12 + meta_size]; - PackRpcHeader(header_and_meta, meta_size, payload_size); - ::google::protobuf::io::ArrayOutputStream arr_out(header_and_meta + 12, meta_size); - ::google::protobuf::io::CodedOutputStream coded_out(&arr_out); - meta.SerializeWithCachedSizes(&coded_out); // not calling ByteSize again - CHECK(!coded_out.HadError()); - CHECK_EQ(0, out->append(header_and_meta, sizeof(header_and_meta))); - } else { - char header[12]; - PackRpcHeader(header, meta_size, payload_size); - CHECK_EQ(0, out->append(header, sizeof(header))); - butil::IOBufAsZeroCopyOutputStream buf_stream(out); - ::google::protobuf::io::CodedOutputStream coded_out(&buf_stream); - meta.SerializeWithCachedSizes(&coded_out); - CHECK(!coded_out.HadError()); - } -} - - -// Used by UT, can't be static. -static void SendRpcResponseReuse(int64_t correlation_id, - Controller* cntl, - const google::protobuf::Message* req, - const google::protobuf::Message* res, - const Server* server, - MethodStatus* method_status, - int64_t received_us) { - ControllerPrivateAccessor accessor(cntl); - Span* span = accessor.span(); - if (span) { - span->set_start_send_us(butil::cpuwide_time_us()); - } - Socket* sock = accessor.get_sending_socket(); - - //std::unique_ptr recycle_req(req); - //std::unique_ptr recycle_res(res); - - std::unique_ptr recycle_cntl(cntl); - ConcurrencyRemover concurrency_remover(method_status, cntl, received_us); - - ClosureGuard guard(brpc::NewCallback(cntl, &Controller::CallAfterRpcResp, req, res)); - - StreamId response_stream_id = accessor.response_stream(); - - if (cntl->IsCloseConnection()) { - StreamClose(response_stream_id); - sock->SetFailed(); - return; - } - bool append_body = false; - butil::IOBuf res_body; - // `res' can be NULL here, in which case we don't serialize it - // If user calls `SetFailed' on Controller, we don't serialize - // response either - CompressType type = cntl->response_compress_type(); - if (res != NULL && !cntl->Failed()) { - if (!res->IsInitialized()) { - cntl->SetFailed( - ERESPONSE, "Missing required fields in response: %s", - res->InitializationErrorString().c_str()); - } else if (!SerializeAsCompressedData(*res, &res_body, type)) { - cntl->SetFailed(ERESPONSE, "Fail to serialize response, " - "CompressType=%s", CompressTypeToCStr(type)); - } else { - append_body = true; - } - } - - // Don't use res->ByteSize() since it may be compressed - size_t res_size = 0; - size_t attached_size = 0; - if (append_body) { - res_size = res_body.length(); - attached_size = cntl->response_attachment().length(); - } - - int error_code = cntl->ErrorCode(); - if (error_code == -1) { - // replace general error (-1) with INTERNAL_SERVER_ERROR to make a - // distinction between server error and client error - error_code = EINTERNAL; - } - RpcMeta meta; - RpcResponseMeta* response_meta = meta.mutable_response(); - response_meta->set_error_code(error_code); - if (!cntl->ErrorText().empty()) { - // Only set error_text when it's not empty since protobuf Message - // always new the string no matter if it's empty or not. - response_meta->set_error_text(cntl->ErrorText()); - } - meta.set_correlation_id(correlation_id); - meta.set_compress_type(cntl->response_compress_type()); - if (attached_size > 0) { - meta.set_attachment_size(attached_size); - } - SocketUniquePtr stream_ptr; - if (response_stream_id != INVALID_STREAM_ID) { - if (Socket::Address(response_stream_id, &stream_ptr) == 0) { - Stream* s = (Stream*)stream_ptr->conn(); - s->FillSettings(meta.mutable_stream_settings()); - s->SetHostSocket(sock); - } else { - LOG(WARNING) << "Stream=" << response_stream_id - << " was closed before sending response"; - } - } - - if (cntl->has_response_user_fields() && - !cntl->response_user_fields()->empty()) { - ::google::protobuf::Map& user_fields - = *meta.mutable_user_fields(); - user_fields.insert(cntl->response_user_fields()->begin(), - cntl->response_user_fields()->end()); - - } - - butil::IOBuf res_buf; - SerializeRpcHeaderAndMeta(&res_buf, meta, res_size + attached_size); - if (append_body) { - res_buf.append(res_body.movable()); - if (attached_size) { - res_buf.append(cntl->response_attachment().movable()); - } - } - - if (span) { - span->set_response_size(res_buf.size()); - } - // Send rpc response over stream even if server side failed to create - // stream for some reason. - if(cntl->has_remote_stream()){ - // Send the response over stream to notify that this stream connection - // is successfully built. - // Response_stream can be INVALID_STREAM_ID when error occurs. - if (SendStreamData(sock, &res_buf, - accessor.remote_stream_settings()->stream_id(), - accessor.response_stream()) != 0) { - const int errcode = errno; - std::string error_text = butil::string_printf(64, "Fail to write into %s", - sock->description().c_str()); - PLOG_IF(WARNING, errcode != EPIPE) << error_text; - cntl->SetFailed(errcode, "%s", error_text.c_str()); - if(stream_ptr) { - ((Stream*)stream_ptr->conn())->Close(errcode, "%s", - error_text.c_str()); - } - return; - } - - if(stream_ptr) { - // Now it's ok the mark this server-side stream as connected as all the - // written user data would follower the RPC response. - ((Stream*)stream_ptr->conn())->SetConnected(); - } - } else{ - // Have the risk of unlimited pending responses, in which case, tell - // users to set max_concurrency. - Socket::WriteOptions wopt; - wopt.ignore_eovercrowded = true; - if (sock->Write(&res_buf, &wopt) != 0) { - const int errcode = errno; - PLOG_IF(WARNING, errcode != EPIPE) << "Fail to write into " << *sock; - cntl->SetFailed(errcode, "Fail to write into %s", - sock->description().c_str()); - return; - } - } - - if (span) { - // TODO: this is not sent - span->set_sent_us(butil::cpuwide_time_us()); - } -} - -// Used by other protocols as well. -void EndRunningCallMethodInPool( - ::google::protobuf::Service* service, - const ::google::protobuf::MethodDescriptor* method, - ::google::protobuf::RpcController* controller, - const ::google::protobuf::Message* request, - ::google::protobuf::Message* response, - ::google::protobuf::Closure* done); - -static void ProcessRpcRequestReused(InputMessageBase* msg_base) { - const int64_t start_parse_us = butil::cpuwide_time_us(); - DestroyingPtr msg(static_cast(msg_base)); - SocketUniquePtr socket_guard(msg->ReleaseSocket()); - Socket* socket = socket_guard.get(); - const Server* server = static_cast(msg_base->arg()); - ScopedNonServiceError non_service_error(server); - - RpcMeta meta; - if (!ParsePbFromIOBuf(&meta, msg->meta)) { - LOG(WARNING) << "Fail to parse RpcMeta from " << *socket; - socket->SetFailed(EREQUEST, "Fail to parse RpcMeta from %s", - socket->description().c_str()); - return; - } - const RpcRequestMeta &request_meta = meta.request(); - - SampledRequest* sample = AskToBeSampled(); - if (sample) { - sample->meta.set_service_name(request_meta.service_name()); - sample->meta.set_method_name(request_meta.method_name()); - sample->meta.set_compress_type((CompressType)meta.compress_type()); - sample->meta.set_protocol_type(PROTOCOL_BAIDU_STD); - sample->meta.set_attachment_size(meta.attachment_size()); - sample->meta.set_authentication_data(meta.authentication_data()); - sample->request = msg->payload; - sample->submit(start_parse_us); - } - - std::unique_ptr cntl(new (std::nothrow) Controller); - if (NULL == cntl.get()) { - LOG(WARNING) << "Fail to new Controller"; - return; - } - - struct NoDelete { - void operator()(::google::protobuf::Message*) {} - }; - std::unique_ptr req; - std::unique_ptr res; - babylon::ReusableRPCProtocol::Closure* done = nullptr; - - ServerPrivateAccessor server_accessor(server); - ControllerPrivateAccessor accessor(cntl.get()); - const bool security_mode = server->options().security_mode() && - socket->user() == server_accessor.acceptor(); - if (request_meta.has_log_id()) { - cntl->set_log_id(request_meta.log_id()); - } - if (request_meta.has_request_id()) { - cntl->set_request_id(request_meta.request_id()); - } - if (request_meta.has_timeout_ms()) { - cntl->set_timeout_ms(request_meta.timeout_ms()); - } - cntl->set_request_compress_type((CompressType)meta.compress_type()); - accessor.set_server(server) - .set_security_mode(security_mode) - .set_peer_id(socket->id()) - .set_remote_side(socket->remote_side()) - .set_local_side(socket->local_side()) - .set_auth_context(socket->auth_context()) - .set_request_protocol(PROTOCOL_BAIDU_STD) - .set_begin_time_us(msg->received_us()) - .move_in_server_receiving_sock(socket_guard); - - if (meta.has_stream_settings()) { - accessor.set_remote_stream_settings(meta.release_stream_settings()); - } - - if (!meta.user_fields().empty()) { - for (const auto& it : meta.user_fields()) { - (*cntl->request_user_fields())[it.first] = it.second; - } - } - - // Tag the bthread with this server's key for thread_local_data(). - if (server->thread_local_options().thread_local_data_factory) { - bthread_assign_data((void*)&server->thread_local_options()); - } - - Span* span = NULL; - if (IsTraceable(request_meta.has_trace_id())) { - span = Span::CreateServerSpan( - request_meta.trace_id(), request_meta.span_id(), - request_meta.parent_span_id(), msg->base_real_us()); - accessor.set_span(span); - span->set_log_id(request_meta.log_id()); - span->set_remote_side(cntl->remote_side()); - span->set_protocol(PROTOCOL_BAIDU_STD); - span->set_received_us(msg->received_us()); - span->set_start_parse_us(start_parse_us); - span->set_request_size(msg->payload.size() + msg->meta.size() + 12); - } - - MethodStatus* method_status = NULL; - do { - if (!server->IsRunning()) { - cntl->SetFailed(ELOGOFF, "Server is stopping"); - break; - } - - if (socket->is_overcrowded()) { - cntl->SetFailed(EOVERCROWDED, "Connection to %s is overcrowded", - butil::endpoint2str(socket->remote_side()).c_str()); - break; - } - - if (!server_accessor.AddConcurrency(cntl.get())) { - cntl->SetFailed( - ELIMIT, "Reached server's max_concurrency=%d", - server->options().max_concurrency); - break; - } - - if (FLAGS_usercode_in_pthread && TooManyUserCode()) { - cntl->SetFailed(ELIMIT, "Too many user code to run when" - " -usercode_in_pthread is on"); - break; - } - - // NOTE(gejun): jprotobuf sends service names without packages. So the - // name should be changed to full when it's not. - butil::StringPiece svc_name(request_meta.service_name()); - if (svc_name.find('.') == butil::StringPiece::npos) { - const Server::ServiceProperty* sp = - server_accessor.FindServicePropertyByName(svc_name); - if (NULL == sp) { - cntl->SetFailed(ENOSERVICE, "Fail to find service=%s", - request_meta.service_name().c_str()); - break; - } - svc_name = sp->service->GetDescriptor()->full_name(); - } - const Server::MethodProperty* mp = - server_accessor.FindMethodPropertyByFullName( - svc_name, request_meta.method_name()); - if (NULL == mp) { - cntl->SetFailed(ENOMETHOD, "Fail to find method=%s/%s", - request_meta.service_name().c_str(), - request_meta.method_name().c_str()); - break; - } else if (mp->service->GetDescriptor() - == BadMethodService::descriptor()) { - BadMethodRequest breq; - BadMethodResponse bres; - breq.set_service_name(request_meta.service_name()); - mp->service->CallMethod(mp->method, cntl.get(), &breq, &bres, NULL); - break; - } - // Switch to service-specific error. - non_service_error.release(); - method_status = mp->status; - if (method_status) { - int rejected_cc = 0; - if (!method_status->OnRequested(&rejected_cc, cntl.get())) { - cntl->SetFailed(ELIMIT, "Rejected by %s's ConcurrencyLimiter, concurrency=%d", - mp->method->full_name().c_str(), rejected_cc); - break; - } - } - google::protobuf::Service* svc = mp->service; - const google::protobuf::MethodDescriptor* method = mp->method; - accessor.set_method(method); - - - if (!server->AcceptRequest(cntl.get())) { - break; - } - - if (span) { - span->ResetServerSpanName(method->full_name()); - } - const int req_size = static_cast(msg->payload.size()); - butil::IOBuf req_buf; - butil::IOBuf* req_buf_ptr = &msg->payload; - if (meta.has_attachment_size()) { - if (req_size < meta.attachment_size()) { - cntl->SetFailed(EREQUEST, - "attachment_size=%d is larger than request_size=%d", - meta.attachment_size(), req_size); - break; - } - int body_without_attachment_size = req_size - meta.attachment_size(); - msg->payload.cutn(&req_buf, body_without_attachment_size); - req_buf_ptr = &req_buf; - cntl->request_attachment().swap(msg->payload); - } - - CompressType req_cmp_type = (CompressType)meta.compress_type(); - done = babylon::ReusableRPCProtocol::create(mp); - done->set_correlation_id(meta.correlation_id()); - done->set_received_us(msg->received_us()); - done->set_server(server); - done->set_controller(cntl.get()); - req.reset(done->request()); - if (!ParseFromCompressedData(*req_buf_ptr, req.get(), req_cmp_type)) { - cntl->SetFailed(EREQUEST, "Fail to parse request message, " - "CompressType=%s, request_size=%d", - CompressTypeToCStr(req_cmp_type), req_size); - break; - } - - res.reset(done->response()); - // `socket' will be held until response has been sent - //google::protobuf::Closure* done = ::brpc::NewCallback< - // int64_t, Controller*, const google::protobuf::Message*, - // const google::protobuf::Message*, const Server*, - // MethodStatus*, int64_t>( - // &SendRpcResponse, meta.correlation_id(), cntl.get(), - // req.get(), res.get(), server, - // method_status, msg->received_us()); - - // optional, just release resource ASAP - msg.reset(); - req_buf.clear(); - - if (span) { - span->set_start_callback_us(butil::cpuwide_time_us()); - span->AsParent(); - } - if (!FLAGS_usercode_in_pthread) { - return svc->CallMethod(method, cntl.release(), - req.release(), res.release(), done); - } - if (BeginRunningUserCode()) { - svc->CallMethod(method, cntl.release(), - req.release(), res.release(), done); - return EndRunningUserCodeInPlace(); - } else { - return EndRunningCallMethodInPool( - svc, method, cntl.release(), - req.release(), res.release(), done); - } - } while (false); - - // `cntl', `req' and `res' will be deleted inside `SendRpcResponse' - // `socket' will be held until response has been sent - if (done != nullptr) { - done->Run(); - } else { - SendRpcResponseReuse(meta.correlation_id(), cntl.release(), - req.release(), res.release(), server, - method_status, msg->received_us()); - } -} - -} // namespace policy -} // namespace brpc - -// clang-format on -// 以上内容主体拷贝自 -// https://github.com/apache/brpc/blob/1.9.0/src/brpc/policy/baidu_rpc_protocol.cpp -// 修改其中正反序列化相关部分 -//////////////////////////////////////////////////////////////////////////////// - -DECLARE_bool(babylon_rpc_full_reuse); - -BABYLON_NAMESPACE_BEGIN - -ReusableRPCProtocol::ReusableRPCProtocol(const char* protocol_name) noexcept { - parse = ::brpc::policy::ParseRpcMessage; - serialize_request = nullptr; - pack_request = nullptr; - process_request = ::brpc::policy::ProcessRpcRequestReused; - verify = ::brpc::policy::VerifyRpcRequest; - parse_server_address = nullptr; - get_method_name = nullptr; - supported_connection_type = ::brpc::CONNECTION_TYPE_ALL; - name = protocol_name; -} - -void ReusableRPCProtocol::Closure::Run() noexcept { - ::brpc::policy::SendRpcResponseReuse(_correlation_id, _controller, _request, - _response, _server, _method_status, - _received_us); - if (FLAGS_babylon_rpc_full_reuse) { - _manager.clear(); - } else { - resource().release(); - } - _pool->push(::std::unique_ptr {this}); -} - -BABYLON_NAMESPACE_END diff --git a/example/use-arena-with-brpc/server.cpp b/example/use-arena-with-brpc/server.cpp index 77fea9f3..86ea8f4a 100644 --- a/example/use-arena-with-brpc/server.cpp +++ b/example/use-arena-with-brpc/server.cpp @@ -3,10 +3,20 @@ #include "butil/logging.h" #include "echo.pb.h" #include "gflags/gflags.h" -#include "reusable_rpc_protocol.h" + +#if WITH_BABYLON +#include "swiss_message_factory.h" +#endif + +#include "tcmalloc/malloc_extension.h" + +#include DEFINE_int32(port, 8000, "TCP Port of this server"); -DEFINE_bool(use_arena, false, "use arena allocate request and response"); +DEFINE_int32(mode, 0, + "0: Use DefaultRpcPBMessageFactory\n" + "1: Use ArenaRpcPBMessageFactory\n" + "2: Use SwissRpcPBMessageFactory\n"); namespace example { class EchoServiceImpl : public EchoService { @@ -27,42 +37,47 @@ class EchoServiceImpl : public EchoService { } // namespace example int main(int argc, char* argv[]) { - // Parse gflags. We recommend you to use gflags as well. - gflags::ParseCommandLineFlags(&argc, &argv, true); - - if (0 != ::babylon::ReusableRPCProtocol::register_protocol()) { - LOG(ERROR) << "register ReusableRPCProtocol failed"; - return -1; - } + ::gflags::ParseCommandLineFlags(&argc, &argv, true); - // Generally you only need one Server. - brpc::Server server; - - // Instance of your service. - example::EchoServiceImpl echo_service_impl; - - // Add the service into server. Notice the second parameter, because the - // service is put on stack, we don't want server to delete it, otherwise - // use brpc::SERVER_OWNS_SERVICE. + ::brpc::Server server; + ::example::EchoServiceImpl echo_service_impl; if (server.AddService(&echo_service_impl, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { LOG(ERROR) << "Fail to add service"; return -1; } - butil::EndPoint point = butil::EndPoint(butil::IP_ANY, FLAGS_port); - - // Start the server. - brpc::ServerOptions options; - if (FLAGS_use_arena) { - options.enabled_protocols = "baidu_std_reuse"; + ::butil::EndPoint point = ::butil::EndPoint(butil::IP_ANY, FLAGS_port); + ::brpc::ServerOptions options; + if (FLAGS_mode == 0) { + LOG(INFO) << "Use DefaultRpcPBMessageFactory"; + } else if (FLAGS_mode == 1) { + LOG(INFO) << "Use ArenaRpcPBMessageFactory"; + options.rpc_pb_message_factory = ::brpc::GetArenaRpcPBMessageFactory(); +#if WITH_BABYLON + } else if (FLAGS_mode == 2) { + LOG(INFO) << "Use SwissRpcPBMessageFactory"; + auto factory = new SwissRpcPBMessageFactory; + factory->set_page_size(4096); + factory->set_free_page_capacity(2048); + factory->set_free_message_capacity(128); + options.rpc_pb_message_factory = factory; +#endif + } else { + LOG(ERROR) << "Invalid mode " << FLAGS_mode; + return -1; } + if (server.Start(point, &options) != 0) { LOG(ERROR) << "Fail to start EchoServer"; return -1; } - // Wait until Ctrl-C is pressed, then Stop() and Join() the server. + ::std::thread([] { + ::tcmalloc::MallocExtension::SetBackgroundReleaseRate( + ::tcmalloc::MallocExtension::BytesPerSecond(16 << 10)); + ::tcmalloc::MallocExtension::ProcessBackgroundActions(); + }).detach(); server.RunUntilAskedToQuit(); return 0; } diff --git a/example/use-arena-with-brpc/swiss_message_factory.cpp b/example/use-arena-with-brpc/swiss_message_factory.cpp new file mode 100644 index 00000000..2db282df --- /dev/null +++ b/example/use-arena-with-brpc/swiss_message_factory.cpp @@ -0,0 +1,79 @@ +#include "swiss_message_factory.h" + +SwissRpcPBMessageFactory::SwissRpcPBMessageFactory() noexcept { + _cached_page_allocator.set_upstream(_new_delete_page_allocator); +} + +void SwissRpcPBMessageFactory::set_page_size(size_t page_size) noexcept { + _new_delete_page_allocator.set_page_size(page_size); +} + +void SwissRpcPBMessageFactory::set_free_page_capacity( + size_t free_page_capacity) noexcept { + _cached_page_allocator.set_free_page_capacity(free_page_capacity); +} + +void SwissRpcPBMessageFactory::set_free_message_capacity( + size_t free_message_capacity) noexcept { + _free_message_capacity = free_message_capacity; + _pool.set_creator([&]() -> ::std::unique_ptr { + auto messages = new Messages; + messages->set_page_allocator(_cached_page_allocator); + return {messages, {}}; + }); + _pool.set_recycler([](Messages& messages) { + messages.clear(); + }); + _pool.reserve_and_clear(_free_message_capacity); +} + +::brpc::RpcPBMessages* SwissRpcPBMessageFactory::Get( + const Service& service, const MethodDescriptor& method) { + auto messages = _pool.pop().release(); + messages->prepare(service, method); + messages->set_pool(_pool); + return messages; +} + +void SwissRpcPBMessageFactory::Return(::brpc::RpcPBMessages* messages) { + auto reusable_messages = static_cast(messages); + reusable_messages->pool().push( + ::std::unique_ptr {reusable_messages}); +} + +SwissRpcPBMessageFactory::Message* +SwissRpcPBMessageFactory::Messages::Request() { + return _request; +} + +SwissRpcPBMessageFactory::Message* +SwissRpcPBMessageFactory::Messages::Response() { + return _response; +} + +void SwissRpcPBMessageFactory::Messages::set_page_allocator( + ::babylon::PageAllocator& page_allocator) noexcept { + _resource.set_page_allocator(page_allocator); +} + +void SwissRpcPBMessageFactory::Messages::set_pool( + ::babylon::ObjectPool& pool) noexcept { + _pool = &pool; +} + +::babylon::ObjectPool& +SwissRpcPBMessageFactory::Messages::pool() noexcept { + return *_pool; +} + +void SwissRpcPBMessageFactory::Messages::prepare( + const Service& service, const MethodDescriptor& method) noexcept { + using Arena = ::google::protobuf::Arena; + auto arena = &static_cast(_resource); + _request = service.GetRequestPrototype(&method).New(arena); + _response = service.GetResponsePrototype(&method).New(arena); +} + +void SwissRpcPBMessageFactory::Messages::clear() noexcept { + _resource.release(); +} diff --git a/example/use-arena-with-brpc/swiss_message_factory.h b/example/use-arena-with-brpc/swiss_message_factory.h new file mode 100644 index 00000000..2d8a0a34 --- /dev/null +++ b/example/use-arena-with-brpc/swiss_message_factory.h @@ -0,0 +1,57 @@ +#include "babylon/concurrent/object_pool.h" +#include "babylon/concurrent/transient_hash_table.h" +#include "babylon/reusable/manager.h" + +#include "brpc/rpc_pb_message_factory.h" + +class SwissRpcPBMessageFactory : public ::brpc::RpcPBMessageFactory { + private: + using Message = ::google::protobuf::Message; + using Service = ::google::protobuf::Service; + using MethodDescriptor = ::google::protobuf::MethodDescriptor; + class Messages; + + public: + SwissRpcPBMessageFactory() noexcept; + SwissRpcPBMessageFactory(SwissRpcPBMessageFactory&&) = delete; + SwissRpcPBMessageFactory(const SwissRpcPBMessageFactory&) = delete; + SwissRpcPBMessageFactory& operator=(SwissRpcPBMessageFactory&&) = delete; + SwissRpcPBMessageFactory& operator=(const SwissRpcPBMessageFactory&) = delete; + + void set_page_size(size_t page_size) noexcept; + void set_free_page_capacity(size_t free_page_capacity) noexcept; + void set_free_message_capacity(size_t free_message_capacity) noexcept; + + private: + virtual ::brpc::RpcPBMessages* Get(const Service& service, + const MethodDescriptor& method) override; + virtual void Return(::brpc::RpcPBMessages* messages) override; + + size_t _free_message_capacity {128}; + + ::babylon::ConcurrentTransientHashMap> + _pool_for_method {32}; + ::babylon::NewDeletePageAllocator _new_delete_page_allocator; + ::babylon::CachedPageAllocator _cached_page_allocator; + ::babylon::ObjectPool _pool; +}; + +class SwissRpcPBMessageFactory::Messages : public ::brpc::RpcPBMessages { + public: + void set_page_allocator(::babylon::PageAllocator& page_allocator) noexcept; + void set_pool(::babylon::ObjectPool& pool) noexcept; + ::babylon::ObjectPool& pool() noexcept; + + void prepare(const Service& service, const MethodDescriptor& method) noexcept; + void clear() noexcept; + + virtual Message* Request(); + virtual Message* Response(); + + private: + ::babylon::ObjectPool* _pool {nullptr}; + Message* _request {nullptr}; + Message* _response {nullptr}; + ::babylon::SwissMemoryResource _resource; +}; diff --git a/example/use-async-logger/MODULE.bazel b/example/use-async-logger/MODULE.bazel index 41c9a6a2..9b68ef6f 100644 --- a/example/use-async-logger/MODULE.bazel +++ b/example/use-async-logger/MODULE.bazel @@ -1,5 +1,12 @@ -bazel_dep(name = 'babylon', version = '1.4.1') -bazel_dep(name = 'brpc', version = '1.10.0.bcr.1') +bazel_dep(name = 'babylon', version = '1.4.2') +bazel_dep(name = 'brpc', version = '1.11.0') bazel_dep(name = 'spdlog', version = '1.14.1') bazel_dep(name = 'tcmalloc', version = '0.0.0-20240411-5ed309d') -single_version_override(module_name = 'protobuf', version = '3.19.6') + +single_version_override(module_name = 'rules_fuzzing', version = '0.5.1') +single_version_override(module_name = 'protobuf', version = '28.3') +archive_override(module_name = 'brpc', + urls = ['https://github.com/apache/brpc/archive/30a56dd153010c0be88d7c197712e5b95843ff9c.zip'], + integrity = 'sha256-94aE0H4NZo72DSB0wtq6YlMZc9fOpGAlK4QHgG0FvXg=', + strip_prefix = 'brpc-30a56dd153010c0be88d7c197712e5b95843ff9c', +) diff --git a/example/use-counter-with-bvar/MODULE.bazel b/example/use-counter-with-bvar/MODULE.bazel index 7a90c868..4b811476 100644 --- a/example/use-counter-with-bvar/MODULE.bazel +++ b/example/use-counter-with-bvar/MODULE.bazel @@ -1,4 +1,11 @@ -bazel_dep(name = 'babylon', version = '1.4.1') -bazel_dep(name = 'brpc', version = '1.10.0.bcr.1') +bazel_dep(name = 'babylon', version = '1.4.2') +bazel_dep(name = 'brpc', version = '1.11.0') bazel_dep(name = 'tcmalloc', version = '0.0.0-20240411-5ed309d') -single_version_override(module_name = 'protobuf', version = '3.19.6') + +single_version_override(module_name = 'rules_fuzzing', version = '0.5.1') +single_version_override(module_name = 'protobuf', version = '28.3') +archive_override(module_name = 'brpc', + urls = ['https://github.com/apache/brpc/archive/30a56dd153010c0be88d7c197712e5b95843ff9c.zip'], + integrity = 'sha256-94aE0H4NZo72DSB0wtq6YlMZc9fOpGAlK4QHgG0FvXg=', + strip_prefix = 'brpc-30a56dd153010c0be88d7c197712e5b95843ff9c', +) diff --git a/example/use-execution-queue/MODULE.bazel b/example/use-execution-queue/MODULE.bazel index 7a90c868..4b811476 100644 --- a/example/use-execution-queue/MODULE.bazel +++ b/example/use-execution-queue/MODULE.bazel @@ -1,4 +1,11 @@ -bazel_dep(name = 'babylon', version = '1.4.1') -bazel_dep(name = 'brpc', version = '1.10.0.bcr.1') +bazel_dep(name = 'babylon', version = '1.4.2') +bazel_dep(name = 'brpc', version = '1.11.0') bazel_dep(name = 'tcmalloc', version = '0.0.0-20240411-5ed309d') -single_version_override(module_name = 'protobuf', version = '3.19.6') + +single_version_override(module_name = 'rules_fuzzing', version = '0.5.1') +single_version_override(module_name = 'protobuf', version = '28.3') +archive_override(module_name = 'brpc', + urls = ['https://github.com/apache/brpc/archive/30a56dd153010c0be88d7c197712e5b95843ff9c.zip'], + integrity = 'sha256-94aE0H4NZo72DSB0wtq6YlMZc9fOpGAlK4QHgG0FvXg=', + strip_prefix = 'brpc-30a56dd153010c0be88d7c197712e5b95843ff9c', +) diff --git a/example/use-with-bthread/MODULE.bazel b/example/use-with-bthread/MODULE.bazel index 1f09265d..5c049453 100644 --- a/example/use-with-bthread/MODULE.bazel +++ b/example/use-with-bthread/MODULE.bazel @@ -1,2 +1,2 @@ -bazel_dep(name = 'babylon', version = '1.4.1') -bazel_dep(name = 'brpc', version = '1.10.0.bcr.1') +bazel_dep(name = 'babylon', version = '1.4.2') +bazel_dep(name = 'brpc', version = '1.11.0') diff --git a/example/use-with-glog/MODULE.bazel b/example/use-with-glog/MODULE.bazel index e503f838..b0d62864 100644 --- a/example/use-with-glog/MODULE.bazel +++ b/example/use-with-glog/MODULE.bazel @@ -1,2 +1,2 @@ -bazel_dep(name = 'babylon', version = '1.4.1') +bazel_dep(name = 'babylon', version = '1.4.2') bazel_dep(name = 'glog', version = '0.7.1') diff --git a/registry/modules/protobuf/28.3.arenastring/patches/arenastring.patch b/registry/modules/protobuf/28.3.arenastring/patches/arenastring.patch index e6f3d4d2..fa8b7bfe 100644 --- a/registry/modules/protobuf/28.3.arenastring/patches/arenastring.patch +++ b/registry/modules/protobuf/28.3.arenastring/patches/arenastring.patch @@ -33,20 +33,24 @@ index 000000000..8d1c8b69c +++ b/WORKSPACE.bzlmod @@ -0,0 +1 @@ + +diff --git a/src/file_lists.cmake b/src/file_lists.cmake +index 07152d92d..200c67cf6 100644 +--- a/src/file_lists.cmake ++++ b/src/file_lists.cmake +@@ -243,6 +243,8 @@ set(libprotobuf_lite_hdrs + ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_allocation_policy.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_cleanup.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/arenastring.h ++ # ARENASTRING PATCH: add new file ++ ${protobuf_SOURCE_DIR}/src/google/protobuf/arenastring_impl.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/arenaz_sampler.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor_lite.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/endian.h diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel -index 381e0e24e..7c495fa98 100644 +index 381e0e24e..d56a5836c 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel -@@ -464,6 +464,8 @@ cc_library( - srcs = [ - "any_lite.cc", - "arenastring.cc", -+ # ARENASTRING PATCH: add new file -+ "arenastring_impl.cc", - "arenaz_sampler.cc", - "extension_set.cc", - "generated_enum_util.cc", -@@ -483,6 +485,8 @@ cc_library( +@@ -483,6 +483,8 @@ cc_library( "any.h", "arena.h", "arenastring.h", @@ -55,7 +59,7 @@ index 381e0e24e..7c495fa98 100644 "arenaz_sampler.h", "descriptor_lite.h", "endian.h", -@@ -823,6 +827,9 @@ filegroup( +@@ -823,6 +825,9 @@ filegroup( "map_proto2_unittest.proto", "map_unittest.proto", "unittest.proto", @@ -65,7 +69,7 @@ index 381e0e24e..7c495fa98 100644 "unittest_custom_options.proto", "unittest_embed_optimize_for.proto", "unittest_empty.proto", -@@ -838,6 +845,9 @@ filegroup( +@@ -838,6 +843,9 @@ filegroup( "unittest_optimize_for.proto", "unittest_proto3.proto", "unittest_proto3_arena.proto", @@ -75,7 +79,7 @@ index 381e0e24e..7c495fa98 100644 "unittest_proto3_arena_lite.proto", "unittest_proto3_bad_macros.proto", "unittest_proto3_extensions.proto", -@@ -1225,6 +1235,7 @@ cc_test( +@@ -1225,6 +1233,7 @@ cc_test( ], ) @@ -83,7 +87,7 @@ index 381e0e24e..7c495fa98 100644 cc_test( name = "arenastring_unittest", srcs = ["arenastring_unittest.cc"], -@@ -1241,6 +1252,23 @@ cc_test( +@@ -1241,6 +1250,23 @@ cc_test( ], ) @@ -448,6 +452,49 @@ index fb730a416..8f2bc8e1d 100644 } inline void Mixin::set_allocated_root(std::string* value) { ::google::protobuf::internal::TSanWrite(&_impl_); +diff --git a/src/google/protobuf/arena_align_test.cc b/src/google/protobuf/arena_align_test.cc +index 7e0347f10..fa95d47ac 100644 +--- a/src/google/protobuf/arena_align_test.cc ++++ b/src/google/protobuf/arena_align_test.cc +@@ -147,7 +147,11 @@ TEST(ArenaAlign, Padded) { + } + + TEST(ArenaAlign, CeilPtr) { +- alignas(64) char p[129] = {0}; ++ struct alignas(64) S { ++ char p[129] = {0}; ++ }; ++ auto s = ::std::make_unique(); ++ auto p = s->p; + auto align_64 = ArenaAlignAs(64); + EXPECT_THAT(align_64.Ceil(p + 0), Eq(p)); + EXPECT_THAT(align_64.Ceil(p + 1), Eq(p + 64)); +@@ -159,7 +163,11 @@ TEST(ArenaAlign, CeilPtr) { + } + + TEST(ArenaAlign, CheckAligned) { +- alignas(128) char p[129] = {0}; ++ struct alignas(64) S { ++ char p[129] = {0}; ++ }; ++ auto s = ::std::make_unique(); ++ auto p = s->p; + auto align_64 = ArenaAlignAs(64); + EXPECT_THAT(align_64.CheckAligned(p + 0), Eq(p)); + EXPECT_THAT(align_64.CheckAligned(p + 64), Eq(p + 64)); +@@ -178,7 +186,11 @@ TEST(ArenaAlign, CheckAligned) { + } + + TEST(ArenaAlign, CeilDefaultAligned) { +- alignas(128) char p[129] = {0}; ++ struct alignas(64) S { ++ char p[129] = {0}; ++ }; ++ auto s = ::std::make_unique(); ++ auto p = s->p; + auto align_64 = ArenaAlignAs(64); + EXPECT_THAT(align_64.CeilDefaultAligned(p + 0), Eq(p)); + EXPECT_THAT(align_64.CeilDefaultAligned(p + 8), Eq(p + 64)); diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index 857abf057..9500344e3 100644 --- a/src/google/protobuf/arena_unittest.cc @@ -759,173 +806,12 @@ index 0c1c4e271..54ff21e61 100644 } inline std::string* ArenaStringPtr::UnsafeMutablePointer() { -diff --git a/src/google/protobuf/arenastring_impl.cc b/src/google/protobuf/arenastring_impl.cc -new file mode 100644 -index 000000000..c4a976bbe ---- /dev/null -+++ b/src/google/protobuf/arenastring_impl.cc -@@ -0,0 +1,155 @@ -+#include "google/protobuf/arenastring_impl.h" -+ -+#include "google/protobuf/port_def.inc" -+ -+namespace google { -+namespace protobuf { -+ -+ArenaStringAccessor& ArenaStringAccessor::assign(const_pointer data, -+ size_type size) noexcept { -+ auto* buffer = qualified_buffer(size); -+ set_size(size); -+ __builtin_memcpy(buffer, data, size); -+ buffer[size] = '\0'; -+ return *this; -+} -+ -+void ArenaStringAccessor::reserve(size_type required_capacity) noexcept { -+ if (required_capacity > capacity()) { -+ auto origin_size = size(); -+ recreate_buffer(required_capacity); -+ set_size_and_terminator(origin_size); -+ } -+} -+ -+void ArenaStringAccessor::push_back(value_type c) noexcept { -+ auto origin_size = size(); -+ auto* buffer = qualified_buffer(origin_size + 1, origin_size << 1); -+ set_size(origin_size + 1); -+ buffer[origin_size] = c; -+ buffer[origin_size + 1] = '\0'; -+} -+ -+ArenaStringAccessor& ArenaStringAccessor::append( -+ const_pointer append_data, size_type append_size) noexcept { -+ auto origin_size = size(); -+ auto* buffer = qualified_buffer(origin_size + append_size); -+ set_size(origin_size + append_size); -+ __builtin_memcpy(buffer + origin_size, append_data, append_size); -+ buffer[origin_size + append_size] = '\0'; -+ return *this; -+} -+ -+void ArenaStringAccessor::swap(ArenaStringAccessor other) noexcept { -+#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI -+ auto* tmp = representation().data; -+ *reinterpret_cast(_ptr) = other.representation().data; -+ *reinterpret_cast(other._ptr) = tmp; -+#else // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+ _ptr->swap(*other._ptr); -+#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+} -+ -+void ArenaStringAccessor::resize(size_type new_size, value_type c) noexcept { -+ auto origin_size = size(); -+ auto buffer = qualified_buffer(new_size); -+ set_size_and_terminator(new_size); -+ if (new_size > origin_size) { -+ __builtin_memset(buffer + origin_size, c, new_size - origin_size); -+ } -+} -+ -+internal::StdStringRep& ArenaStringAccessor::representation() noexcept { -+#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI -+ return *(*reinterpret_cast(_ptr) - 1); -+#else // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+ return *reinterpret_cast(_ptr); -+#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+} -+ -+ArenaStringAccessor::pointer ArenaStringAccessor::recreate_buffer( -+ size_type capacity) noexcept { -+#if __GLIBCXX__ -+ size_t buffer_size = capacity + 1; -+ buffer_size = (buffer_size + 7) & static_cast(-8); -+ capacity = buffer_size - 1; -+#if _GLIBCXX_USE_CXX11_ABI -+ auto* buffer = reinterpret_cast(_arena->AllocateAligned(buffer_size)); -+ __builtin_memcpy(buffer, data(), size()); -+ auto& rep = representation(); -+ rep.data = buffer; -+ rep.capacity = capacity; -+ return buffer; -+#else // !_GLIBCXX_USE_CXX11_ABI -+ auto* rep = reinterpret_cast( -+ _arena->AllocateAligned(sizeof(StdStringRep) + buffer_size)); -+ rep->capacity = capacity; -+ rep->refcount = -1; -+ __builtin_memcpy(rep->data, data(), size()); -+ *reinterpret_cast(_ptr) = rep->data; -+ return rep->data; -+#endif // !_GLIBCXX_USE_CXX11_ABI -+#else // !__GLIBCXX__ -+ capacity = (capacity + 16) & static_cast(-16); -+ auto* buffer = reinterpret_cast(_arena->AllocateAligned(capacity)); -+ __builtin_memcpy(buffer, data(), size()); -+ auto& rep = representation(); -+ rep.long_format.data = buffer; -+ rep.long_format.capacity = capacity + 1; -+ return rep.long_format.data; -+#endif // !__GLIBCXX__ -+} -+ -+ArenaStringAccessor::pointer ArenaStringAccessor::writable_buffer() noexcept { -+#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI -+ return representation().refcount <= 0 ? representation().data : &(*_ptr)[0]; -+#else // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+ return &(*_ptr)[0]; -+#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+} -+ -+void ArenaStringAccessor::set_size(size_type size) noexcept { -+#if __GLIBCXX__ -+ auto& rep = representation(); -+ rep.size = size; -+#else // !__GLIBCXX__ -+ auto& rep = representation(); -+ if (rep.is_long()) { -+ rep.long_format.size = size; -+ } else { -+ rep.shot_format.size = size << 1; -+ } -+#endif // !__GLIBCXX__ -+} -+ -+void ArenaStringAccessor::set_size_and_terminator(size_type size) noexcept { -+#if __GLIBCXX__ -+ auto& rep = representation(); -+ rep.size = size; -+ rep.data[size] = '\0'; -+#else // !__GLIBCXX__ -+ auto& rep = representation(); -+ if (rep.is_long()) { -+ rep.long_format.size = size; -+ rep.long_format.data[size] = '\0'; -+ } else { -+ rep.shot_format.size = size << 1; -+ rep.shot_format.data[size] = '\0'; -+ } -+#endif // !__GLIBCXX__ -+} -+ -+void MaybeArenaStringAccessor::clear() noexcept { -+#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI -+ if (representation().refcount <= 0) { -+ ArenaStringAccessor::clear(); -+ return; -+ } -+#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI -+ underlying()->clear(); -+} -+ -+} // namespace protobuf -+} // namespace google -+ -+#include "google/protobuf/port_undef.inc" diff --git a/src/google/protobuf/arenastring_impl.h b/src/google/protobuf/arenastring_impl.h new file mode 100644 -index 000000000..05324e276 +index 000000000..140d280f6 --- /dev/null +++ b/src/google/protobuf/arenastring_impl.h -@@ -0,0 +1,522 @@ +@@ -0,0 +1,907 @@ +#pragma once + +// Feature check macro @@ -1011,159 +897,485 @@ index 000000000..05324e276 + using reverse_iterator = ::std::reverse_iterator; + using const_reverse_iterator = ::std::reverse_iterator; + -+ using StdStringRep = internal::StdStringRep; -+ -+ // Disable default constructor and copy constructor ++ // Disable default constructor and default operator= + ArenaStringAccessor() = delete; + inline ArenaStringAccessor(ArenaStringAccessor&&) noexcept = default; + inline ArenaStringAccessor(const ArenaStringAccessor&) noexcept = default; + ArenaStringAccessor& operator=(ArenaStringAccessor&&) = delete; + ArenaStringAccessor& operator=(const ArenaStringAccessor&) = delete; -+ ~ArenaStringAccessor() noexcept = default; ++ inline ~ArenaStringAccessor() noexcept = default; + + // Assign -+ inline ArenaStringAccessor& operator=(::absl::string_view other) noexcept { -+ return assign(other.data(), other.size()); -+ } -+ inline ArenaStringAccessor& assign(::absl::string_view other) noexcept { -+ return assign(other.data(), other.size()); -+ } -+ ArenaStringAccessor& assign(const_pointer data, size_type size) noexcept; ++ inline ArenaStringAccessor& operator=(::absl::string_view other) noexcept; ++ inline ArenaStringAccessor& assign(::absl::string_view other) noexcept; ++ inline ArenaStringAccessor& assign(const_pointer data, ++ size_type size) noexcept; + + // Element access -+ inline reference operator[](size_type position) noexcept { -+ return writable_buffer()[position]; -+ } -+ inline const_reference operator[](size_type position) const noexcept { -+ return data()[position]; -+ } -+ inline const_pointer data() const noexcept { return c_str(); } -+ inline const_pointer c_str() const noexcept { -+ return static_cast(_ptr)->c_str(); -+ } -+ inline operator ::absl::string_view() const noexcept { -+ return ::absl::string_view(data(), size()); -+ } -+ inline operator const ::std::string&() const noexcept { return *_ptr; } ++ inline reference operator[](size_type position) noexcept; ++ inline const_reference operator[](size_type position) const noexcept; ++ inline const_pointer data() const noexcept; ++ inline const_pointer c_str() const noexcept; ++ inline operator ::absl::string_view() const noexcept; ++ inline operator const ::std::string&() const noexcept; + + // Iterators -+ inline iterator begin() noexcept { return iterator(writable_buffer()); } -+ inline const_iterator cbegin() const noexcept { -+ return const_iterator(data()); -+ } -+ inline const_iterator end() noexcept { -+ return iterator(writable_buffer() + size()); -+ } -+ inline const_iterator cend() const noexcept { -+ return const_iterator(data() + size()); -+ } ++ inline iterator begin() noexcept; ++ inline const_iterator begin() const noexcept; ++ inline const_iterator cbegin() const noexcept; ++ inline iterator end() noexcept; ++ inline const_iterator end() const noexcept; ++ inline const_iterator cend() const noexcept; + + // Capacity -+ inline bool empty() const noexcept { return _ptr->empty(); } -+ inline size_type size() const noexcept { return _ptr->size(); } -+ void reserve(size_type required_capacity) noexcept; -+ inline size_type capacity() const noexcept { return _ptr->capacity(); } ++ inline bool empty() const noexcept; ++ inline size_type size() const noexcept; ++ inline void reserve(size_type required_capacity) noexcept; ++ inline size_type capacity() const noexcept; + + // Modifiers -+ inline void clear() noexcept { set_size_and_terminator(0); } -+ void push_back(value_type c) noexcept; -+ inline ArenaStringAccessor& append(::absl::string_view sv) noexcept { -+ return append(sv.data(), sv.size()); -+ } -+ ArenaStringAccessor& append(const_pointer append_data, -+ size_type append_size) noexcept; -+ inline ArenaStringAccessor& operator+=(char ch) noexcept { -+ push_back(ch); -+ return *this; -+ } -+ inline ArenaStringAccessor& operator+=(::absl::string_view sv) noexcept { -+ return append(sv.data(), sv.size()); -+ } -+ void resize(size_type new_size) noexcept { resize(new_size, '\0'); } -+ void resize(size_type new_size, value_type c) noexcept; -+ void swap(ArenaStringAccessor other) noexcept; ++ inline void clear() noexcept; ++ inline void push_back(value_type c) noexcept; ++ inline ArenaStringAccessor& append(::absl::string_view sv) noexcept; ++ inline ArenaStringAccessor& append(const_pointer append_data, ++ size_type append_size) noexcept; ++ inline ArenaStringAccessor& operator+=(char ch) noexcept; ++ inline ArenaStringAccessor& operator+=(::absl::string_view sv) noexcept; ++ inline void resize(size_type new_size) noexcept; ++ inline void resize(size_type new_size, value_type c) noexcept; ++ inline void swap(ArenaStringAccessor other) noexcept; + + // Operations -+ inline int compare(::absl::string_view other) const noexcept { -+ return static_cast<::absl::string_view>(*this).compare(other); -+ } ++ inline int compare(::absl::string_view other) const noexcept; + + //////////////////////////////////////////////////////////////////////////// -+ // Special function -+ inline static ArenaStringAccessor create(Arena* arena) noexcept { -+ auto* ptr = reinterpret_cast<::std::string*>( -+ arena->AllocateAligned(sizeof(::std::string))); -+ new (ptr)::std::string(); -+ return ArenaStringAccessor(arena, ptr); ++ // Special functions ++ // Create string on arena ++ inline static ArenaStringAccessor create(Arena* arena) noexcept; ++ template ++ inline static ArenaStringAccessor create(Arena* arena, T&& value) noexcept; ++ // Swap strings both on **same** arena or both not on arena ++ inline static void swap(::std::string* left, ::std::string* right) noexcept; ++ // Wrap constructor ++ inline ArenaStringAccessor(Arena* arena, ::std::string* ptr) noexcept; ++ inline Arena* arena() const noexcept; ++ inline ::std::string* underlying() const noexcept; ++ // Support absl::strings_internal::STLStringResizeUninitialized ++ inline char* __resize_default_init(size_type new_size) noexcept; ++ //////////////////////////////////////////////////////////////////////////// ++ ++ private: ++ inline internal::StdStringRep& representation() noexcept; ++ ++ inline pointer qualified_buffer(size_type required_capacity) noexcept; ++ inline pointer qualified_buffer(size_type required_capacity, ++ size_type predict_capacity) noexcept; ++ inline pointer writable_buffer() noexcept; ++ inline pointer recreate_buffer(size_type capacity) noexcept; ++ ++ inline void set_size(size_type size) noexcept; ++ inline void set_size_and_terminator(size_type size) noexcept; ++ ++ Arena* _arena{nullptr}; ++ ::std::string* _ptr{nullptr}; ++}; ++ ++class MaybeArenaStringAccessor : public ArenaStringAccessor { ++ public: ++ using ArenaStringAccessor::ArenaStringAccessor; ++ inline MaybeArenaStringAccessor(const ArenaStringAccessor& other) noexcept; ++ ++ // Assign ++ template ++ inline MaybeArenaStringAccessor& operator=(T&& other) noexcept; ++ // Deal with assign string specially. Try to keep copy on write state when ++ // using old abi. ++ inline MaybeArenaStringAccessor& operator=( ++ const ::std::string& other) noexcept; ++ inline MaybeArenaStringAccessor& operator=(::std::string& other) noexcept; ++ inline MaybeArenaStringAccessor& operator=(::std::string&& other) noexcept; ++ // Support api of ArenaStringPtr and InlinedStringField ++ template ++ inline MaybeArenaStringAccessor& operator=( ++ ::std::reference_wrapper other) noexcept; ++ template ++ inline MaybeArenaStringAccessor& assign(T&& other) noexcept; ++ inline MaybeArenaStringAccessor& assign(const_pointer data, ++ size_type size) noexcept; ++ inline MaybeArenaStringAccessor& assign(const ::std::string& other) noexcept; ++ inline MaybeArenaStringAccessor& assign(::std::string& other) noexcept; ++ inline MaybeArenaStringAccessor& assign(::std::string&& other) noexcept; ++ template ++ inline MaybeArenaStringAccessor& assign( ++ ::std::reference_wrapper other) noexcept; ++ ++ // Element access ++ inline reference operator[](size_type position) noexcept; ++ using ArenaStringAccessor::operator[]; ++ using ArenaStringAccessor::c_str; ++ using ArenaStringAccessor::data; ++ using ArenaStringAccessor::operator ::absl::string_view; ++ using ArenaStringAccessor::operator const ::std::string&; ++ ++ // Iterators ++ inline iterator begin() noexcept; ++ using ArenaStringAccessor::begin; ++ using ArenaStringAccessor::cbegin; ++ inline iterator end() noexcept; ++ using ArenaStringAccessor::cend; ++ using ArenaStringAccessor::end; ++ ++ // Capacity ++ using ArenaStringAccessor::empty; ++ using ArenaStringAccessor::size; ++ inline void reserve(size_type required_capacity) noexcept; ++ using ArenaStringAccessor::capacity; ++ ++ // Modifiers ++ inline void clear() noexcept; ++ inline void push_back(value_type c) noexcept; ++ inline MaybeArenaStringAccessor& append(::absl::string_view sv) noexcept; ++ inline MaybeArenaStringAccessor& append(const_pointer append_data, ++ size_type append_size) noexcept; ++ inline MaybeArenaStringAccessor& operator+=(char ch) noexcept; ++ inline MaybeArenaStringAccessor& operator+=(::absl::string_view sv) noexcept; ++ inline void resize(size_type new_size) noexcept; ++ inline void resize(size_type new_size, value_type c) noexcept; ++ using ArenaStringAccessor::swap; ++ ++ //////////////////////////////////////////////////////////////////////////// ++ // Special functions ++ // Create string maybe on arena ++ inline static MaybeArenaStringAccessor create(Arena* arena) noexcept; ++ template ++ inline static MaybeArenaStringAccessor create(Arena* arena, ++ T&& value) noexcept; ++ // Clear string maybe on arena ++ inline static void clear(::std::string* ptr) noexcept; ++ // Wrap constructor ++ inline MaybeArenaStringAccessor(::std::string* string) noexcept; ++ using ArenaStringAccessor::arena; ++ using ArenaStringAccessor::underlying; ++ // Support absl::strings_internal::STLStringResizeUninitialized ++ inline void __resize_default_init(size_type new_size) noexcept; ++ // Make operator* and operator-> both to self to imitate a string* ++ inline MaybeArenaStringAccessor* operator->() noexcept; ++ inline const MaybeArenaStringAccessor* operator->() const noexcept; ++ inline MaybeArenaStringAccessor& operator*() noexcept; ++ inline const MaybeArenaStringAccessor& operator*() const noexcept; ++ // Destroy string if not on arena ++ inline void destroy() noexcept; ++ // Also support absl::Format(MaybeArenaStringAccessor, ...) ++ inline operator ::absl::FormatRawSink() noexcept; ++ //////////////////////////////////////////////////////////////////////////// ++ ++ private: ++ // Support absl::Format(MaybeArenaStringAccessor*, ...) ++ friend inline void AbslFormatFlush(MaybeArenaStringAccessor* accessor, ++ ::absl::string_view sv) noexcept { ++ accessor->append(sv.data(), sv.size()); ++ } ++}; ++ ++#if GOOGLE_PROTOBUF_MUTABLE_DONATED_STRING ++using MutableStringType = MaybeArenaStringAccessor; ++using MutableStringReferenceType = MaybeArenaStringAccessor; ++#else // !GOOGLE_PROTOBUF_MUTABLE_DONATED_STRING ++using MutableStringType = ::std::string*; ++using MutableStringReferenceType = ::std::string&; ++#endif // !GOOGLE_PROTOBUF_MUTABLE_DONATED_STRING ++ ++//////////////////////////////////////////////////////////////////////////////// ++// ArenaStringAccessor begin ++inline ArenaStringAccessor& ArenaStringAccessor::operator=( ++ ::absl::string_view other) noexcept { ++ return assign(other.data(), other.size()); ++} ++ ++inline ArenaStringAccessor& ArenaStringAccessor::assign( ++ ::absl::string_view other) noexcept { ++ return assign(other.data(), other.size()); ++} ++ ++inline ArenaStringAccessor& ArenaStringAccessor::assign( ++ const_pointer data, size_type size) noexcept { ++ auto buffer = qualified_buffer(size); ++ set_size(size); ++ __builtin_memcpy(buffer, data, size); ++ buffer[size] = '\0'; ++ return *this; ++} ++ ++inline ArenaStringAccessor::reference ArenaStringAccessor::operator[]( ++ size_type position) noexcept { ++ return writable_buffer()[position]; ++} ++ ++inline ArenaStringAccessor::const_reference ArenaStringAccessor::operator[]( ++ size_type position) const noexcept { ++ return data()[position]; ++} ++ ++inline ArenaStringAccessor::const_pointer ArenaStringAccessor::data() ++ const noexcept { ++ return c_str(); ++} ++ ++inline ArenaStringAccessor::const_pointer ArenaStringAccessor::c_str() ++ const noexcept { ++ return static_cast(_ptr)->c_str(); ++} ++ ++inline ArenaStringAccessor::operator ::absl::string_view() const noexcept { ++ return ::absl::string_view(data(), size()); ++} ++ ++inline ArenaStringAccessor::operator const ::std::string&() const noexcept { ++ return *_ptr; ++} ++ ++inline ArenaStringAccessor::iterator ArenaStringAccessor::begin() noexcept { ++ return iterator(writable_buffer()); ++} ++ ++inline ArenaStringAccessor::const_iterator ArenaStringAccessor::begin() ++ const noexcept { ++ return cbegin(); ++} ++ ++inline ArenaStringAccessor::const_iterator ArenaStringAccessor::cbegin() ++ const noexcept { ++ return const_iterator(data()); ++} ++ ++inline ArenaStringAccessor::iterator ArenaStringAccessor::end() noexcept { ++ return iterator(writable_buffer() + size()); ++} ++ ++inline ArenaStringAccessor::const_iterator ArenaStringAccessor::end() ++ const noexcept { ++ return cend(); ++} ++ ++inline ArenaStringAccessor::const_iterator ArenaStringAccessor::cend() ++ const noexcept { ++ return const_iterator(data() + size()); ++} ++ ++inline bool ArenaStringAccessor::empty() const noexcept { ++ return _ptr->empty(); ++} ++ ++inline ArenaStringAccessor::size_type ArenaStringAccessor::size() ++ const noexcept { ++ return _ptr->size(); ++} ++ ++inline void ArenaStringAccessor::reserve(size_type required_capacity) noexcept { ++ if (required_capacity > capacity()) { ++ auto origin_size = size(); ++ recreate_buffer(required_capacity); ++ set_size_and_terminator(origin_size); ++ } ++} ++ ++inline ArenaStringAccessor::size_type ArenaStringAccessor::capacity() ++ const noexcept { ++ return _ptr->capacity(); ++} ++ ++inline void ArenaStringAccessor::clear() noexcept { ++ set_size_and_terminator(0); ++} ++ ++inline void ArenaStringAccessor::push_back(value_type c) noexcept { ++ auto origin_size = size(); ++ auto buffer = qualified_buffer(origin_size + 1, origin_size << 1); ++ set_size(origin_size + 1); ++ buffer[origin_size] = c; ++ buffer[origin_size + 1] = '\0'; ++} ++ ++inline ArenaStringAccessor& ArenaStringAccessor::append( ++ ::absl::string_view sv) noexcept { ++ return append(sv.data(), sv.size()); ++} ++ ++inline ArenaStringAccessor& ArenaStringAccessor::append( ++ const_pointer append_data, size_type append_size) noexcept { ++ auto origin_size = size(); ++ auto buffer = qualified_buffer(origin_size + append_size); ++ set_size(origin_size + append_size); ++ __builtin_memcpy(buffer + origin_size, append_data, append_size); ++ buffer[origin_size + append_size] = '\0'; ++ return *this; ++} ++ ++inline ArenaStringAccessor& ArenaStringAccessor::operator+=(char ch) noexcept { ++ push_back(ch); ++ return *this; ++} ++ ++inline ArenaStringAccessor& ArenaStringAccessor::operator+=( ++ ::absl::string_view sv) noexcept { ++ return append(sv.data(), sv.size()); ++} ++ ++inline void ArenaStringAccessor::resize(size_type new_size) noexcept { ++ resize(new_size, '\0'); ++} ++ ++inline void ArenaStringAccessor::resize(size_type new_size, ++ value_type c) noexcept { ++ auto origin_size = size(); ++ auto buffer = qualified_buffer(new_size); ++ set_size_and_terminator(new_size); ++ if (new_size > origin_size) { ++ __builtin_memset(buffer + origin_size, c, new_size - origin_size); + } ++} + -+ template -+ inline static ArenaStringAccessor create(Arena* arena, T&& value) noexcept { -+ return create(arena) = ::std::forward(value); -+ } ++inline void ArenaStringAccessor::swap(ArenaStringAccessor other) noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ auto tmp = representation().data; ++ *reinterpret_cast(_ptr) = other.representation().data; ++ *reinterpret_cast(other._ptr) = tmp; ++#else // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ _ptr->swap(*other._ptr); ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++} + -+ // Clear function dont need arena -+ inline static void clear(::std::string* ptr) noexcept { -+ ArenaStringAccessor(nullptr, ptr).clear(); -+ } ++inline int ArenaStringAccessor::compare( ++ ::absl::string_view other) const noexcept { ++ return static_cast<::absl::string_view>(*this).compare(other); ++} + -+ // Wwap function dont need arena -+ // but left and right must both on same arena -+ inline static void swap(::std::string* left, ::std::string* right) noexcept { -+ ArenaStringAccessor(nullptr, left) -+ .swap(ArenaStringAccessor(nullptr, right)); -+ } ++inline ArenaStringAccessor ArenaStringAccessor::create(Arena* arena) noexcept { ++ auto ptr = reinterpret_cast<::std::string*>( ++ arena->AllocateAligned(sizeof(::std::string))); ++ new (ptr)::std::string(); ++ return ArenaStringAccessor(arena, ptr); ++} + -+ // Wrapper construct -+ inline ArenaStringAccessor(Arena* arena, ::std::string* ptr) noexcept -+ : _arena(arena), _ptr(ptr) {} -+ inline Arena* arena() const noexcept { return _arena; } -+ inline ::std::string* underlying() const noexcept { return _ptr; } ++template ++inline ArenaStringAccessor ArenaStringAccessor::create(Arena* arena, ++ T&& value) noexcept { ++ return create(arena) = ::std::forward(value); ++} + -+ // Support absl::strings_internal::STLStringResizeUninitialized -+ inline char* __resize_default_init(size_type new_size) noexcept { -+ auto buffer = qualified_buffer(new_size); -+ set_size_and_terminator(new_size); -+ return buffer; -+ } ++inline void ArenaStringAccessor::swap(::std::string* left, ++ ::std::string* right) noexcept { ++ ArenaStringAccessor(nullptr, left).swap(ArenaStringAccessor(nullptr, right)); ++} + -+ // Also support absl::Format(ArenaStringAccessor, ...) -+ inline operator ::absl::FormatRawSink() noexcept { -+ return ::absl::FormatRawSink(this); -+ } -+ //////////////////////////////////////////////////////////////////////////// ++inline ArenaStringAccessor::ArenaStringAccessor(Arena* arena, ++ ::std::string* ptr) noexcept ++ : _arena(arena), _ptr(ptr) {} + -+ protected: -+ StdStringRep& representation() noexcept; ++inline Arena* ArenaStringAccessor::arena() const noexcept { return _arena; } ++inline ::std::string* ArenaStringAccessor::underlying() const noexcept { ++ return _ptr; ++} + -+ pointer recreate_buffer(size_type capacity) noexcept; ++inline char* ArenaStringAccessor::__resize_default_init( ++ size_type new_size) noexcept { ++ auto buffer = qualified_buffer(new_size); ++ set_size_and_terminator(new_size); ++ return buffer; ++} + -+ pointer writable_buffer() noexcept; ++inline internal::StdStringRep& ArenaStringAccessor::representation() noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ return *(*reinterpret_cast(_ptr) - 1); ++#else // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ return *reinterpret_cast(_ptr); ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++} + -+ inline pointer qualified_buffer(size_type required_capacity, -+ size_type predict_capacity) noexcept { -+ return required_capacity <= capacity() ? writable_buffer() -+ : recreate_buffer(predict_capacity); -+ } ++inline ArenaStringAccessor::pointer ArenaStringAccessor::qualified_buffer( ++ size_type required_capacity, size_type predict_capacity) noexcept { ++ return required_capacity <= capacity() ? writable_buffer() ++ : recreate_buffer(predict_capacity); ++} + -+ inline pointer qualified_buffer(size_type required_capacity) noexcept { -+ return qualified_buffer(required_capacity, required_capacity); -+ } ++inline ArenaStringAccessor::pointer ArenaStringAccessor::qualified_buffer( ++ size_type required_capacity) noexcept { ++ return qualified_buffer(required_capacity, required_capacity); ++} + -+ void set_size(size_type size) noexcept; ++inline ArenaStringAccessor::pointer ++ArenaStringAccessor::writable_buffer() noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ return representation().data; ++#else // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ return &(*_ptr)[0]; ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++} + -+ void set_size_and_terminator(size_type size) noexcept; ++inline ArenaStringAccessor::pointer ArenaStringAccessor::recreate_buffer( ++ size_type capacity) noexcept { ++#if __GLIBCXX__ ++ size_t buffer_size = capacity + 1; ++ buffer_size = (buffer_size + 7) & static_cast(-8); ++ capacity = buffer_size - 1; ++#if _GLIBCXX_USE_CXX11_ABI ++ auto buffer = reinterpret_cast(_arena->AllocateAligned(buffer_size)); ++ __builtin_memcpy(buffer, data(), size()); ++ auto& rep = representation(); ++ rep.data = buffer; ++ rep.capacity = capacity; ++ return buffer; ++#else // !_GLIBCXX_USE_CXX11_ABI ++ auto rep = reinterpret_cast( ++ _arena->AllocateAligned(sizeof(StdStringRep) + buffer_size)); ++ rep->capacity = capacity; ++ rep->refcount = -1; ++ __builtin_memcpy(rep->data, data(), size()); ++ *reinterpret_cast(_ptr) = rep->data; ++ return rep->data; ++#endif // !_GLIBCXX_USE_CXX11_ABI ++#else // !__GLIBCXX__ ++ capacity = (capacity + 16) & static_cast(-16); ++ auto buffer = reinterpret_cast(_arena->AllocateAligned(capacity)); ++ __builtin_memcpy(buffer, data(), size()); ++ auto& rep = representation(); ++ rep.long_format.data = buffer; ++ rep.long_format.capacity = capacity + 1; ++ return rep.long_format.data; ++#endif // !__GLIBCXX__ ++} + -+ private: -+ // Support absl::Format(ArenaStringAccessor*, ...) -+ friend inline void AbslFormatFlush(ArenaStringAccessor* accessor, -+ ::absl::string_view sv) noexcept { -+ accessor->append(sv.data(), sv.size()); ++inline void ArenaStringAccessor::set_size(size_type size) noexcept { ++#if __GLIBCXX__ ++ auto& rep = representation(); ++ rep.size = size; ++#else // !__GLIBCXX__ ++ auto& rep = representation(); ++ if (rep.is_long()) { ++ rep.long_format.size = size; ++ } else { ++ rep.shot_format.size = size << 1; + } ++#endif // !__GLIBCXX__ ++} + -+ Arena* _arena; -+ ::std::string* _ptr; -+}; ++inline void ArenaStringAccessor::set_size_and_terminator( ++ size_type size) noexcept { ++#if __GLIBCXX__ ++ auto& rep = representation(); ++ rep.size = size; ++ rep.data[size] = '\0'; ++#else // !__GLIBCXX__ ++ auto& rep = representation(); ++ if (rep.is_long()) { ++ rep.long_format.size = size; ++ rep.long_format.data[size] = '\0'; ++ } else { ++ rep.shot_format.size = size << 1; ++ rep.shot_format.data[size] = '\0'; ++ } ++#endif // !__GLIBCXX__ ++} + +inline bool operator==(const ArenaStringAccessor& left, + const ArenaStringAccessor& right) noexcept { @@ -1254,195 +1466,254 @@ index 000000000..05324e276 + ::absl::string_view right) noexcept { + return *left.underlying() >= right; +} ++// ArenaStringAccessor end ++//////////////////////////////////////////////////////////////////////////////// + -+class MaybeArenaStringAccessor : public ArenaStringAccessor { -+ public: -+ using ArenaStringAccessor::ArenaStringAccessor; -+ -+ MaybeArenaStringAccessor(const ArenaStringAccessor& other) noexcept -+ : ArenaStringAccessor(other) {} ++//////////////////////////////////////////////////////////////////////////////// ++// MaybeArenaStringAccessor begin ++inline MaybeArenaStringAccessor::MaybeArenaStringAccessor( ++ const ArenaStringAccessor& other) noexcept ++ : ArenaStringAccessor(other) {} + -+ // Assign -+ template -+ inline MaybeArenaStringAccessor& operator=(T&& other) { -+ return assign(::std::forward(other)); -+ } -+ template -+ inline MaybeArenaStringAccessor& assign(T&& other) { -+ ::absl::string_view sv(::std::forward(other)); -+ return assign(sv.data(), sv.size()); -+ } -+ inline MaybeArenaStringAccessor& assign(const_pointer data, size_type size) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::assign(data, size); -+ } else { -+ underlying()->assign(data, size); -+ } -+ return *this; -+ } -+ // Deal with assign string specially. Try to keep copy on write state when -+ // using old abi -+ inline MaybeArenaStringAccessor& operator=(const ::std::string& other) { -+ return assign(other); -+ } -+ inline MaybeArenaStringAccessor& assign(const ::std::string& other) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::assign(other); -+ } else { -+ underlying()->assign(other); -+ } -+ return *this; -+ } -+ inline MaybeArenaStringAccessor& operator=(::std::string& other) { -+ return assign(static_cast(other)); -+ } -+ inline MaybeArenaStringAccessor& assign(::std::string& other) { -+ return assign(static_cast(other)); -+ } -+ inline MaybeArenaStringAccessor& operator=(::std::string&& other) { -+ return assign(::std::move(other)); -+ } -+ inline MaybeArenaStringAccessor& assign(::std::string&& other) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::assign(other); -+ } else { -+ underlying()->assign(::std::move(other)); -+ } -+ return *this; -+ } -+ template -+ inline MaybeArenaStringAccessor& operator=( -+ ::std::reference_wrapper other) { -+ return assign(other); ++template ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator=( ++ T&& other) noexcept { ++ return assign(::std::forward(other)); ++} ++template ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::assign( ++ T&& other) noexcept { ++ ::absl::string_view sv(::std::forward(other)); ++ return assign(sv.data(), sv.size()); ++} ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::assign( ++ const_pointer data, size_type size) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::assign(data, size); ++ } else { ++ underlying()->assign(data, size); + } -+ template -+ inline MaybeArenaStringAccessor& assign(::std::reference_wrapper other) { -+ return assign(other.get()); ++ return *this; ++} ++ ++// Deal with assign string specially. Try to keep copy on write state when ++// using old abi ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator=( ++ const ::std::string& other) noexcept { ++ return assign(other); ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::assign( ++ const ::std::string& other) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::assign(other); ++ } else { ++ underlying()->assign(other); + } ++ return *this; ++} + -+ inline void reserve(size_type required_capacity) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::reserve(required_capacity); -+ } else if (required_capacity > capacity()) { -+ underlying()->reserve(required_capacity); -+ } ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator=( ++ ::std::string& other) noexcept { ++ return assign(static_cast(other)); ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::assign( ++ ::std::string& other) noexcept { ++ return assign(static_cast(other)); ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator=( ++ ::std::string&& other) noexcept { ++ return assign(::std::move(other)); ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::assign( ++ ::std::string&& other) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::assign(other); ++ } else { ++ underlying()->assign(::std::move(other)); + } ++ return *this; ++} + -+ void clear() noexcept; ++template ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator=( ++ ::std::reference_wrapper other) noexcept { ++ return assign(other); ++} + -+ inline void push_back(value_type c) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::push_back(c); -+ } else { -+ underlying()->push_back(c); -+ } ++template ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::assign( ++ ::std::reference_wrapper other) noexcept { ++ return assign(other.get()); ++} ++ ++inline MaybeArenaStringAccessor::reference MaybeArenaStringAccessor::operator[]( ++ size_type position) noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ if (representation().refcount >= 0) { ++ return underlying()->operator[](position); + } ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ return ArenaStringAccessor::operator[](position); ++} + -+ inline MaybeArenaStringAccessor& append(::absl::string_view sv) { -+ return append(sv.data(), sv.size()); ++inline MaybeArenaStringAccessor::iterator ++MaybeArenaStringAccessor::begin() noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ if (representation().refcount >= 0) { ++ return underlying()->begin(); + } ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ return ArenaStringAccessor::begin(); ++} + -+ inline MaybeArenaStringAccessor& append(const_pointer data, size_type size) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::append(data, size); -+ } else { -+ underlying()->append(data, size); -+ } -+ return *this; ++inline MaybeArenaStringAccessor::iterator ++MaybeArenaStringAccessor::end() noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ if (representation().refcount >= 0) { ++ return underlying()->end(); + } ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ return ArenaStringAccessor::end(); ++} + -+ inline MaybeArenaStringAccessor& operator+=(char ch) noexcept { -+ push_back(ch); -+ return *this; ++inline void MaybeArenaStringAccessor::reserve( ++ size_type required_capacity) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::reserve(required_capacity); ++ } else if (required_capacity > capacity()) { ++ underlying()->reserve(required_capacity); + } ++} + -+ inline MaybeArenaStringAccessor& operator+=(::absl::string_view sv) noexcept { -+ return append(sv.data(), sv.size()); ++inline void MaybeArenaStringAccessor::clear() noexcept { ++#if __GLIBCXX__ && !_GLIBCXX_USE_CXX11_ABI ++ if (representation().refcount <= 0) { ++ ArenaStringAccessor::clear(); ++ return; + } ++#endif // !__GLIBCXX__ || _GLIBCXX_USE_CXX11_ABI ++ underlying()->clear(); ++} + -+ inline void resize(size_type size) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::resize(size); -+ } else { -+ underlying()->resize(size); -+ } ++inline void MaybeArenaStringAccessor::push_back(value_type c) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::push_back(c); ++ } else { ++ underlying()->push_back(c); + } -+ inline void resize(size_type size, value_type c) { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::resize(size, c); -+ } else { -+ underlying()->resize(size, c); -+ } ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::append( ++ ::absl::string_view sv) noexcept { ++ return append(sv.data(), sv.size()); ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::append( ++ const_pointer data, size_type size) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::append(data, size); ++ } else { ++ underlying()->append(data, size); + } ++ return *this; ++} + -+ //////////////////////////////////////////////////////////////////////////// -+ // Special function -+ inline static MaybeArenaStringAccessor create(Arena* arena) { -+ if (arena != nullptr) { -+ return ArenaStringAccessor::create(arena); -+ } else { -+ return MaybeArenaStringAccessor(new ::std::string); -+ } ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator+=( ++ char ch) noexcept { ++ push_back(ch); ++ return *this; ++} ++ ++inline MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator+=( ++ ::absl::string_view sv) noexcept { ++ return append(sv.data(), sv.size()); ++} ++ ++inline void MaybeArenaStringAccessor::resize(size_type size) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::resize(size); ++ } else { ++ underlying()->resize(size); + } ++} + -+ template -+ inline static MaybeArenaStringAccessor create(Arena* arena, T&& value) { -+ return create(arena) = ::std::forward(value); ++inline void MaybeArenaStringAccessor::resize(size_type size, ++ value_type c) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::resize(size, c); ++ } else { ++ underlying()->resize(size, c); + } ++} + -+ inline static void clear(::std::string* ptr) noexcept { -+ MaybeArenaStringAccessor(ptr).clear(); ++inline MaybeArenaStringAccessor MaybeArenaStringAccessor::create( ++ Arena* arena) noexcept { ++ if (arena != nullptr) { ++ return ArenaStringAccessor::create(arena); ++ } else { ++ return MaybeArenaStringAccessor(new ::std::string); + } ++} + -+ // Add wrapper constructor for normal string -+ inline MaybeArenaStringAccessor(::std::string* string) noexcept -+ : ArenaStringAccessor(nullptr, string) {} -+ using ArenaStringAccessor::arena; -+ using ArenaStringAccessor::underlying; ++template ++inline MaybeArenaStringAccessor MaybeArenaStringAccessor::create( ++ Arena* arena, T&& value) noexcept { ++ return create(arena) = ::std::forward(value); ++} + -+ // Support absl::strings_internal::STLStringResizeUninitialized -+ inline void __resize_default_init(size_type new_size) noexcept { -+ if (arena() != nullptr) { -+ ArenaStringAccessor::__resize_default_init(new_size); -+ } else { -+ ::absl::strings_internal::STLStringResizeUninitialized(underlying(), -+ new_size); -+ } -+ } ++inline void MaybeArenaStringAccessor::clear(::std::string* ptr) noexcept { ++ MaybeArenaStringAccessor(ptr).clear(); ++} + -+ // Make operator* and operator-> both to self to imitate a string* -+ inline MaybeArenaStringAccessor* operator->() { return this; } -+ inline const MaybeArenaStringAccessor* operator->() const { return this; } -+ inline MaybeArenaStringAccessor& operator*() { return *this; } -+ inline const MaybeArenaStringAccessor& operator*() const { return *this; } -+ -+ inline void destroy() noexcept { -+ if (arena() == nullptr) { -+ delete underlying(); -+ } -+ } ++inline MaybeArenaStringAccessor::MaybeArenaStringAccessor( ++ ::std::string* string) noexcept ++ : ArenaStringAccessor(nullptr, string) {} + -+ // Also support absl::Format(MaybeArenaStringAccessor, ...) -+ inline operator ::absl::FormatRawSink() noexcept { -+ return ::absl::FormatRawSink(this); ++inline void MaybeArenaStringAccessor::__resize_default_init( ++ size_type new_size) noexcept { ++ if (arena() != nullptr) { ++ ArenaStringAccessor::__resize_default_init(new_size); ++ } else { ++ ::absl::strings_internal::STLStringResizeUninitialized(underlying(), ++ new_size); + } -+ //////////////////////////////////////////////////////////////////////////// ++} + -+ private: -+ // Support absl::Format -+ friend inline void AbslFormatFlush(MaybeArenaStringAccessor* accessor, -+ ::absl::string_view sv) noexcept { -+ accessor->append(sv.data(), sv.size()); ++inline MaybeArenaStringAccessor* ++MaybeArenaStringAccessor::operator->() noexcept { ++ return this; ++} ++ ++inline const MaybeArenaStringAccessor* MaybeArenaStringAccessor::operator->() ++ const noexcept { ++ return this; ++} ++ ++inline MaybeArenaStringAccessor& ++MaybeArenaStringAccessor::operator*() noexcept { ++ return *this; ++} ++ ++inline const MaybeArenaStringAccessor& MaybeArenaStringAccessor::operator*() ++ const noexcept { ++ return *this; ++} ++ ++inline void MaybeArenaStringAccessor::destroy() noexcept { ++ if (arena() == nullptr) { ++ delete underlying(); + } -+}; ++} + -+#if GOOGLE_PROTOBUF_MUTABLE_DONATED_STRING -+using MutableStringType = MaybeArenaStringAccessor; -+using MutableStringReferenceType = MaybeArenaStringAccessor; -+#else // !GOOGLE_PROTOBUF_MUTABLE_DONATED_STRING -+using MutableStringType = ::std::string*; -+using MutableStringReferenceType = ::std::string&; -+#endif // !GOOGLE_PROTOBUF_MUTABLE_DONATED_STRING ++inline MaybeArenaStringAccessor::operator ::absl::FormatRawSink() noexcept { ++ return ::absl::FormatRawSink(this); ++} ++// MaybeArenaStringAccessor end ++//////////////////////////////////////////////////////////////////////////////// + +} // namespace protobuf +} // namespace google @@ -6258,7 +6529,7 @@ index a17f922b6..1c4a47f28 100644 diff --git a/src/google/protobuf/inlined_string_field.h b/src/google/protobuf/inlined_string_field.h -index d894acb7e..4d2c4972d 100644 +index d894acb7e..06f651d3e 100644 --- a/src/google/protobuf/inlined_string_field.h +++ b/src/google/protobuf/inlined_string_field.h @@ -25,6 +25,8 @@ @@ -6327,6 +6598,15 @@ index d894acb7e..4d2c4972d 100644 } inline const std::string& InlinedStringField::GetNoArena() const { +@@ -429,7 +438,7 @@ inline PROTOBUF_NDEBUG_INLINE void InlinedStringField::InternalSwap( + InlinedStringField* rhs, bool rhs_arena_dtor_registered, + MessageLite* rhs_msg, Arena* arena) { + #ifdef GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL_INLINE +- lhs->get_mutable()->swap(*rhs->get_mutable()); ++ MaybeArenaStringAccessor::swap(lhs->get_mutable(), rhs->get_mutable()); + if (!lhs_arena_dtor_registered && rhs_arena_dtor_registered) { + lhs_msg->OnDemandRegisterArenaDtor(arena); + } else if (lhs_arena_dtor_registered && !rhs_arena_dtor_registered) { @@ -441,7 +450,7 @@ inline PROTOBUF_NDEBUG_INLINE void InlinedStringField::InternalSwap( (void)rhs_arena_dtor_registered; (void)lhs_msg; @@ -6342,15 +6622,15 @@ index d894acb7e..4d2c4972d 100644 (void)donated; - SetNoArena(value); + MutableAccessor(arena, donated) = value; -+} -+ + } + +-inline void InlinedStringField::Set(const char* str, ::google::protobuf::Arena* arena, +inline void InlinedStringField::Set(const std::string& value, Arena* arena, + bool donated, uint32_t* /*donating_states*/, + uint32_t /*mask*/, MessageLite* /*msg*/) { + MutableAccessor(arena, donated) = value; - } - --inline void InlinedStringField::Set(const char* str, ::google::protobuf::Arena* arena, ++} ++ +inline void InlinedStringField::Set(const char* str, + ::google::protobuf::Arena* arena, bool donated, uint32_t* donating_states, diff --git a/registry/modules/protobuf/28.3.arenastring/source.json b/registry/modules/protobuf/28.3.arenastring/source.json index d8995b6f..244a5e01 100644 --- a/registry/modules/protobuf/28.3.arenastring/source.json +++ b/registry/modules/protobuf/28.3.arenastring/source.json @@ -4,6 +4,6 @@ "integrity": "sha256-s7TDts/nS3eurpkJ/DwTAwknF/cbwhVNfBlhrNr1/kw=", "patch_strip": 1, "patches": { - "arenastring.patch": "sha256-vPU3J+tUz3BbyX8fCrmVw9FvG7QYWVKyvVMOiSl1uJk=" + "arenastring.patch": "sha256-xngXoTVCmvrJAtmUKrA1Tu9yrHdkZxL+a+alH10qPZc=" } } diff --git a/src/babylon/concurrent/thread_local.h b/src/babylon/concurrent/thread_local.h index edf82685..9375e536 100644 --- a/src/babylon/concurrent/thread_local.h +++ b/src/babylon/concurrent/thread_local.h @@ -24,8 +24,18 @@ class EnumerableThreadLocal { void set_constructor(::std::function constructor) noexcept; - // 获得线程局部存储 - inline T& local(); + // Get item exclusive to current thread. + inline T& local() noexcept; + + // When calling intensively from single thread, allocation from memory pool + // for example, get item from ConcurrentVector could also significant costly + // than normal thread_local, because of more indirections in segmented vector + // lookup. + // + // Use local_fast instead of local in such scene provide a really really fast + // path. But local_fast may return nullptr when cache is not available, and + // caller should call local again when this happen. + inline T* local_fast() noexcept; // 遍历所有【当前或曾经存在】的线程局部存储 template _storage; size_t _id {fetch_add_id()}; @@ -208,12 +218,23 @@ ABSL_ATTRIBUTE_NOINLINE void EnumerableThreadLocal::set_constructor( } template -ABSL_ATTRIBUTE_ALWAYS_INLINE inline T& EnumerableThreadLocal::local() { - static thread_local Cache cache; - if (ABSL_PREDICT_TRUE(cache.id == _id)) { - return *cache.item; +inline T& EnumerableThreadLocal::local() noexcept { + auto item = local_fast(); + if (ABSL_PREDICT_FALSE(item == nullptr)) { + item = &_storage.ensure(ThreadId::current_thread_id().value); + _s_cache.id = _id; + _s_cache.item = item; + } + return *item; +} + +template +ABSL_ATTRIBUTE_ALWAYS_INLINE inline T* +EnumerableThreadLocal::local_fast() noexcept { + if (ABSL_PREDICT_TRUE(_s_cache.id == _id)) { + return _s_cache.item; } - return local_slow(cache); + return nullptr; } template @@ -224,12 +245,8 @@ EnumerableThreadLocal::fetch_add_id() noexcept { } template -inline T& EnumerableThreadLocal::local_slow(Cache& cache) { - auto& item = _storage.ensure(ThreadId::current_thread_id().value); - cache.id = _id; - cache.item = &item; - return item; -} +thread_local + typename EnumerableThreadLocal::Cache EnumerableThreadLocal::_s_cache; // EnumerableThreadLocal end //////////////////////////////////////////////////////////////////////////////// diff --git a/src/babylon/reusable/memory_resource.cpp b/src/babylon/reusable/memory_resource.cpp index 6cf14dfd..305b3808 100644 --- a/src/babylon/reusable/memory_resource.cpp +++ b/src/babylon/reusable/memory_resource.cpp @@ -82,8 +82,8 @@ ExclusiveMonotonicBufferResource& ExclusiveMonotonicBufferResource::operator=( ::std::swap(_last_page_array, other._last_page_array); ::std::swap(_last_page_pointer, other._last_page_pointer); - ::std::swap(_last_page, other._last_page); - ::std::swap(_last_page_allocated, other._last_page_allocated); + ::std::swap(_free_begin, other._free_begin); + ::std::swap(_free_end, other._free_end); ::std::swap(_space_used, other._space_used); ::std::swap(_space_allocated, other._space_allocated); @@ -142,8 +142,9 @@ void* ExclusiveMonotonicBufferResource::do_allocate_in_new_page( _space_allocated += page_size; if (_last_page_pointer > _last_page_array->pages) { SanitizerHelper::PoisonGuard guard {_last_page_array}; - _last_page = *--_last_page_pointer = page; - _last_page_allocated = bytes; + *--_last_page_pointer = page; + _free_begin = page + bytes; + _free_end = page + page_size; return page; } else { return do_allocate_with_page_in_new_page_array(bytes, page); @@ -155,18 +156,17 @@ void* ExclusiveMonotonicBufferResource::do_allocate_in_new_page( void* ExclusiveMonotonicBufferResource::do_allocate_with_page_in_new_page_array( size_t bytes, char* page) noexcept { auto page_size = _page_allocator->page_size(); - _last_page_allocated = (_last_page_allocated + alignof(PageArray) - 1) & - static_cast(-alignof(PageArray)); - if (_last_page_allocated + sizeof(PageArray) <= page_size) { + do_align(); + if (_free_begin + sizeof(PageArray) <= _free_end) { SanitizerHelper::PoisonGuard last_guard {_last_page_array}; - auto new_page_array = - reinterpret_cast(_last_page + _last_page_allocated); + auto new_page_array = reinterpret_cast(_free_begin); SanitizerHelper::PoisonGuard new_guard {new_page_array}; new_page_array->next = _last_page_array; _last_page_array = new_page_array; _last_page_pointer = &new_page_array->pages[PAGE_ARRAY_CAPACITY - 1]; - _last_page_allocated = bytes; - _last_page = *_last_page_pointer = page; + *_last_page_pointer = page; + _free_begin = page + bytes; + _free_end = page + page_size; } else if (bytes + sizeof(PageArray) <= page_size) { bytes = (bytes + alignof(PageArray) - 1) & static_cast(-alignof(PageArray)); @@ -174,8 +174,9 @@ void* ExclusiveMonotonicBufferResource::do_allocate_with_page_in_new_page_array( new_page_array->next = _last_page_array; _last_page_array = new_page_array; _last_page_pointer = &new_page_array->pages[PAGE_ARRAY_CAPACITY - 1]; - _last_page_allocated = bytes + sizeof(PageArray); - _last_page = *_last_page_pointer = page; + *_last_page_pointer = page; + _free_begin = page + bytes + sizeof(PageArray); + _free_end = page + page_size; } else { auto additional_page = reinterpret_cast(_page_allocator->allocate()); _space_allocated += page_size; @@ -184,8 +185,9 @@ void* ExclusiveMonotonicBufferResource::do_allocate_with_page_in_new_page_array( _last_page_array = new_page_array; new_page_array->pages[PAGE_ARRAY_CAPACITY - 1] = page; _last_page_pointer = &new_page_array->pages[PAGE_ARRAY_CAPACITY - 2]; - _last_page_allocated = sizeof(PageArray); - _last_page = *_last_page_pointer = additional_page; + *_last_page_pointer = additional_page; + _free_begin = additional_page + sizeof(PageArray); + _free_end = additional_page + page_size; SanitizerHelper::poison(additional_page, page_size); } return page; @@ -256,9 +258,9 @@ void ExclusiveMonotonicBufferResource::release() noexcept { } _page_allocator->deallocate(reinterpret_cast(tmp_pages), size); _last_page_pointer = _last_page_array->pages; - _last_page_allocated = UINT32_MAX; + _free_begin = nullptr; + _free_end = nullptr; } - _last_page = nullptr; while (_last_oversize_page_array != nullptr) { SanitizerHelper::unpoison(_last_oversize_page_array); auto iter = _last_oversize_page_pointer; @@ -277,9 +279,10 @@ void ExclusiveMonotonicBufferResource::release() noexcept { bool ExclusiveMonotonicBufferResource::contains(const void* ptr) noexcept { { + auto page_size = _page_allocator->page_size(); auto page_array = _last_page_array; auto iter = _last_page_pointer; - auto size = _last_page_allocated; + auto size = page_size - (_free_end - _free_begin); while (page_array != nullptr) { SanitizerHelper::PoisonGuard guard {page_array}; auto end = &page_array->pages[PAGE_ARRAY_CAPACITY]; @@ -287,7 +290,7 @@ bool ExclusiveMonotonicBufferResource::contains(const void* ptr) noexcept { if (ptr >= *iter && ptr < *iter + size) { return true; } - size = _page_allocator->page_size(); + size = page_size; } page_array = page_array->next; iter = page_array->pages; diff --git a/src/babylon/reusable/memory_resource.h b/src/babylon/reusable/memory_resource.h index 871245bb..b77538d2 100644 --- a/src/babylon/reusable/memory_resource.h +++ b/src/babylon/reusable/memory_resource.h @@ -226,8 +226,8 @@ class ExclusiveMonotonicBufferResource : public MonotonicBufferResource { PageArray* _last_page_array {nullptr}; char** _last_page_pointer {_last_page_array->pages}; - char* _last_page {nullptr}; - size_t _last_page_allocated {UINT32_MAX}; + char* _free_begin {nullptr}; + char* _free_end {nullptr}; size_t _space_used {0}; size_t _space_allocated {0}; @@ -260,6 +260,8 @@ class SharedMonotonicBufferResource : public MonotonicBufferResource { template inline void* allocate(size_t bytes) noexcept; + template + void* allocate_slow(size_t bytes) noexcept; inline void* allocate(size_t bytes, size_t alignment) noexcept; template @@ -447,17 +449,20 @@ ExclusiveMonotonicBufferResource::do_align<1>() noexcept {} inline ABSL_ATTRIBUTE_ALWAYS_INLINE void ExclusiveMonotonicBufferResource::do_align(size_t alignment) noexcept { - _last_page_allocated = - (_last_page_allocated + alignment - 1) & static_cast(-alignment); + auto free_begin = reinterpret_cast(_free_begin); + free_begin = + (free_begin + alignment - 1) & static_cast(-alignment); + _free_begin = reinterpret_cast(free_begin); } inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* ExclusiveMonotonicBufferResource::do_allocate_already_aligned( size_t bytes, size_t alignment) noexcept { _space_used += bytes; - if (_last_page_allocated + bytes <= _page_allocator->page_size()) { - auto result = _last_page + _last_page_allocated; - _last_page_allocated += bytes; + auto result = _free_begin; + auto next = result + bytes; + if (ABSL_PREDICT_TRUE(next <= _free_end)) { + _free_begin = next; return result; } return do_allocate_in_new_page(bytes, alignment); @@ -470,6 +475,16 @@ ExclusiveMonotonicBufferResource::do_allocate_already_aligned( template inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* SharedMonotonicBufferResource::allocate(size_t bytes) noexcept { + auto local = _resources.local_fast(); + if (ABSL_PREDICT_TRUE(local != nullptr)) { + return local->allocate(bytes); + } + return allocate_slow(bytes); +} + +template +ABSL_ATTRIBUTE_NOINLINE void* SharedMonotonicBufferResource::allocate_slow( + size_t bytes) noexcept { return _resources.local().allocate(bytes); }