Skip to content

Commit

Permalink
[Streaming Indexing] Introduce new experimental HTTP transport based …
Browse files Browse the repository at this point in the history
…on Netty 4 and Project Reactor (Reactor Netty) (opensearch-project#9672)

* [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty)

Signed-off-by: Andriy Redko <[email protected]>

* Remove HttpChunk for now since it is not used in non-streaming APIs

Signed-off-by: Andriy Redko <[email protected]>

* Fix postmerge issues

Signed-off-by: Andriy Redko <[email protected]>

---------

Signed-off-by: Andriy Redko <[email protected]>
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
reta authored and shiv0408 committed Apr 25, 2024
1 parent c61b713 commit f3a4509
Show file tree
Hide file tree
Showing 67 changed files with 3,668 additions and 90 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
- Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
- Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
- [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))

### Dependencies
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
Expand Down
4 changes: 4 additions & 0 deletions buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ jna = 5.13.0
netty = 4.1.100.Final
joda = 2.12.2

# project reactor
reactor_netty = 1.1.12
reactor = 3.5.11

# client dependencies
httpclient5 = 5.2.1
httpcore5 = 5.2.2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@
import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
import static org.opensearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;

/**
* The HTTP transport implementations based on Netty 4.
*/
public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
private static final Logger logger = LogManager.getLogger(Netty4HttpServerTransport.class);

Expand Down Expand Up @@ -184,6 +187,17 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
private volatile ServerBootstrap serverBootstrap;
private volatile SharedGroupFactory.SharedGroup sharedGroup;

/**
* Creates new HTTP transport implementations based on Netty 4
* @param settings seetings
* @param networkService network service
* @param bigArrays big array allocator
* @param threadPool thread pool instance
* @param xContentRegistry XContent registry instance
* @param dispatcher dispatcher instance
* @param clusterSettings cluster settings
* @param sharedGroupFactory shared group factory
*/
public Netty4HttpServerTransport(
Settings settings,
NetworkService networkService,
Expand Down

This file was deleted.

21 changes: 0 additions & 21 deletions plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt

This file was deleted.

22 changes: 4 additions & 18 deletions plugins/repository-azure/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,8 @@ dependencies {
api "io.netty:netty-transport-native-unix-common:${versions.netty}"
implementation project(':modules:transport-netty4')
api 'com.azure:azure-storage-blob:12.23.0'
api 'org.reactivestreams:reactive-streams:1.0.4'
api 'io.projectreactor:reactor-core:3.5.6'
api 'io.projectreactor.netty:reactor-netty:1.1.8'
api 'io.projectreactor.netty:reactor-netty-core:1.1.8'
api 'io.projectreactor.netty:reactor-netty-http:1.1.9'
api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
api "org.slf4j:slf4j-api:${versions.slf4j}"
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
Expand Down Expand Up @@ -101,10 +98,6 @@ thirdPartyAudit {
'com.azure.storage.internal.avro.implementation.schema.AvroSchema',
'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
'io.micrometer.context.ContextAccessor',
'io.micrometer.context.ContextRegistry',
'io.micrometer.context.ContextSnapshot',
'io.micrometer.context.ContextSnapshot$Scope',
'io.micrometer.core.instrument.Clock',
'io.micrometer.core.instrument.Counter',
'io.micrometer.core.instrument.Counter$Builder',
'io.micrometer.core.instrument.DistributionSummary',
Expand All @@ -114,14 +107,10 @@ thirdPartyAudit {
'io.micrometer.core.instrument.Meter',
'io.micrometer.core.instrument.MeterRegistry',
'io.micrometer.core.instrument.Metrics',
'io.micrometer.core.instrument.Tag',
'io.micrometer.core.instrument.Tags',
'io.micrometer.core.instrument.Timer',
'io.micrometer.core.instrument.Timer$Builder',
'io.micrometer.core.instrument.Timer$Sample',
'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics',
'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
'io.micrometer.core.instrument.search.Search',
'io.netty.channel.epoll.Epoll',
'io.netty.channel.epoll.EpollDatagramChannel',
'io.netty.channel.epoll.EpollServerSocketChannel',
Expand Down Expand Up @@ -168,9 +157,6 @@ thirdPartyAudit {
'org.slf4j.impl.StaticLoggerBinder',
'org.slf4j.impl.StaticMDCBinder',
'org.slf4j.impl.StaticMarkerBinder',
'reactor.blockhound.BlockHound$Builder',
'reactor.blockhound.integration.BlockHoundIntegration',
'io.micrometer.context.ThreadLocalAccessor',
'io.micrometer.common.KeyValue',
'io.micrometer.common.KeyValues',
'io.micrometer.common.docs.KeyName',
Expand All @@ -190,6 +176,7 @@ thirdPartyAudit {
'io.micrometer.tracing.handler.PropagatingSenderTracingObservationHandler',
'io.micrometer.tracing.propagation.Propagator',
'io.micrometer.core.instrument.observation.MeterObservationHandler',
'io.micrometer.core.instrument.Tags',
'io.micrometer.observation.ObservationHandler',
'io.micrometer.observation.ObservationRegistry',
'io.micrometer.observation.ObservationRegistry$ObservationConfig',
Expand All @@ -210,8 +197,7 @@ thirdPartyAudit {
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1'
)
}

Expand Down

This file was deleted.

21 changes: 0 additions & 21 deletions plugins/repository-azure/licenses/reactive-streams-LICENSE.txt

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
378dc5a375e6440099e837b22cf4b01341cbe4ea

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e839fadb8f45d8a7a2783466faedd03373366c23

This file was deleted.

1 change: 0 additions & 1 deletion plugins/repository-s3/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ dependencies {
api "software.amazon.awssdk:sts:${versions.aws}"
api "software.amazon.awssdk:netty-nio-client:${versions.aws}"

api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
api "commons-logging:commons-logging:${versions.commonslogging}"
Expand Down

This file was deleted.

21 changes: 0 additions & 21 deletions plugins/repository-s3/licenses/reactive-streams-LICENSE.txt

This file was deleted.

Empty file.
Loading

0 comments on commit f3a4509

Please sign in to comment.