[Streaming Indexing] Introduce new experimental HTTP transport based …

…on Netty 4 and Project Reactor (Reactor Netty) (opensearch-project#9672) * [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) Signed-off-by: Andriy Redko <[email protected]> * Remove HttpChunk for now since it is not used in non-streaming APIs Signed-off-by: Andriy Redko <[email protected]> * Fix postmerge issues Signed-off-by: Andriy Redko <[email protected]> --------- Signed-off-by: Andriy Redko <[email protected]> (cherry picked from commit 1782b90)
reta · Nov 6, 2023 · b8d2ed5 · b8d2ed5
1 parent 74d1bb1
commit b8d2ed5
Show file tree

Hide file tree

Showing 66 changed files with 3,675 additions and 83 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
+- [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
 
 ### Dependencies
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -29,6 +29,10 @@ jna               = 5.13.0
 netty             = 4.1.100.Final
 joda              = 2.12.2
 
+# project reactor
+reactor_netty     = 1.1.12
+reactor           = 3.5.11
+
 # client dependencies
 httpclient        = 4.5.14
 httpcore          = 4.4.16

diff --git a/.../transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/.../transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
@@ -99,6 +99,9 @@
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
 import static org.opensearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
 
+/**
+ * The HTTP transport implementations based on Netty 4.
+ */
 public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private static final Logger logger = LogManager.getLogger(Netty4HttpServerTransport.class);
 
@@ -167,6 +170,17 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private volatile ServerBootstrap serverBootstrap;
     private volatile SharedGroupFactory.SharedGroup sharedGroup;
 
+    /**
+     * Creates new HTTP transport implementations based on Netty 4
+     * @param settings seetings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param sharedGroupFactory shared group factory
+     */
     public Netty4HttpServerTransport(
         Settings settings,
         NetworkService networkService,

diff --git a/plugins/crypto-kms/licenses/reactive-streams-1.0.4.jar.sha1 b/plugins/crypto-kms/licenses/reactive-streams-1.0.4.jar.sha1
diff --git a/plugins/discovery-ec2/licenses/reactive-streams-1.0.4.jar.sha1 b/plugins/discovery-ec2/licenses/reactive-streams-1.0.4.jar.sha1
diff --git a/plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt b/plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -56,11 +56,8 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
-  api 'org.reactivestreams:reactive-streams:1.0.3'
-  api 'io.projectreactor:reactor-core:3.5.6'
-  api 'io.projectreactor.netty:reactor-netty:1.1.8'
-  api 'io.projectreactor.netty:reactor-netty-core:1.1.8'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.9'
+  api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
+  api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
@@ -100,7 +97,11 @@ thirdPartyAudit {
     'com.azure.storage.internal.avro.implementation.AvroReaderFactory',
     'com.azure.storage.internal.avro.implementation.schema.AvroSchema',
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
+<<<<<<< HEAD
     'io.micrometer.core.instrument.Clock',
+=======
+    'io.micrometer.context.ContextAccessor',
+>>>>>>> 1782b906a0a ([Streaming Indexing] Introduce new experimental HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) (#9672))
     'io.micrometer.core.instrument.Counter',
     'io.micrometer.core.instrument.Counter$Builder',
     'io.micrometer.core.instrument.DistributionSummary',
@@ -110,14 +111,10 @@ thirdPartyAudit {
     'io.micrometer.core.instrument.Meter',
     'io.micrometer.core.instrument.MeterRegistry',
     'io.micrometer.core.instrument.Metrics',
-    'io.micrometer.core.instrument.Tag',
-    'io.micrometer.core.instrument.Tags',
     'io.micrometer.core.instrument.Timer',
     'io.micrometer.core.instrument.Timer$Builder',
     'io.micrometer.core.instrument.Timer$Sample',
-    'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics',
     'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
-    'io.micrometer.core.instrument.search.Search',
     'io.netty.channel.epoll.Epoll',
     'io.netty.channel.epoll.EpollDatagramChannel',
     'io.netty.channel.epoll.EpollServerSocketChannel',
@@ -164,12 +161,15 @@ thirdPartyAudit {
     'org.slf4j.impl.StaticLoggerBinder',
     'org.slf4j.impl.StaticMDCBinder',
     'org.slf4j.impl.StaticMarkerBinder',
+<<<<<<< HEAD
     'reactor.blockhound.BlockHound$Builder',
     'reactor.blockhound.integration.BlockHoundIntegration',
     'io.micrometer.context.ContextRegistry',
     'io.micrometer.context.ContextSnapshot',
     'io.micrometer.context.ContextSnapshot$Scope',
     'io.micrometer.context.ThreadLocalAccessor',
+=======
+>>>>>>> 1782b906a0a ([Streaming Indexing] Introduce new experimental HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) (#9672))
     'io.micrometer.common.KeyValue',
     'io.micrometer.common.KeyValues',
     'io.micrometer.common.docs.KeyName',
@@ -190,6 +190,7 @@ thirdPartyAudit {
     'io.micrometer.tracing.propagation.Propagator',
     'io.micrometer.context.ContextAccessor',
     'io.micrometer.core.instrument.observation.MeterObservationHandler',
+    'io.micrometer.core.instrument.Tags',
     'io.micrometer.observation.ObservationHandler',
     'io.micrometer.observation.ObservationRegistry',
     'io.micrometer.observation.ObservationRegistry$ObservationConfig',
@@ -210,8 +211,7 @@ thirdPartyAudit {
     'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
-    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
-    'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1'
   )
 }
 

diff --git a/plugins/repository-azure/licenses/reactive-streams-LICENSE.txt b/plugins/repository-azure/licenses/reactive-streams-LICENSE.txt
diff --git a/plugins/repository-azure/licenses/reactor-core-3.5.6.jar.sha1 b/plugins/repository-azure/licenses/reactor-core-3.5.6.jar.sha1
diff --git a/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1
@@ -0,0 +1 @@
+378dc5a375e6440099e837b22cf4b01341cbe4ea
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1
@@ -0,0 +1 @@
+e839fadb8f45d8a7a2783466faedd03373366c23
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.9.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.9.jar.sha1
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
@@ -70,7 +70,6 @@ dependencies {
   api "software.amazon.awssdk:sts:${versions.aws}"
   api "software.amazon.awssdk:netty-nio-client:${versions.aws}"
 
-  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "commons-logging:commons-logging:${versions.commonslogging}"

diff --git a/plugins/repository-s3/licenses/reactive-streams-1.0.4.jar.sha1 b/plugins/repository-s3/licenses/reactive-streams-1.0.4.jar.sha1
diff --git a/plugins/repository-s3/licenses/reactive-streams-LICENSE.txt b/plugins/repository-s3/licenses/reactive-streams-LICENSE.txt
diff --git a/plugins/repository-s3/licenses/reactive-streams-NOTICE.txt b/plugins/repository-s3/licenses/reactive-streams-NOTICE.txt