Merge remote-tracking branch 'origin/1.3' into 1.x

Signed-off-by: Peter Nied <[email protected]>

.github/pull_request_template.md

@@ -11,7 +11,6 @@
   - [ ] New functionality has javadoc added
 - [ ] Failing checks are inspected and point to the corresponding known issue(s) (See: [Troubleshooting Failing Builds](../blob/main/CONTRIBUTING.md#troubleshooting-failing-builds))
 - [ ] Commits are signed per the DCO using --signoff
-- [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/publish-maven-snapshots.yml

@@ -0,0 +1,39 @@
+name: Publish snapshots to maven
+
+on:
+  workflow_dispatch:
+  push:
+      branches:
+        - main
+        - '[0-9]+.[0-9]+'
+        - '[0-9]+.x'
+
+jobs:
+  build-and-publish-snapshots:
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 11
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
+          aws-region: us-east-1
+
+      - name: Publish snapshots to maven
+        run: |
+          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
+          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
+          echo "::add-mask::$SONATYPE_USERNAME"
+          echo "::add-mask::$SONATYPE_PASSWORD"
+          ./gradlew publishNebulaPublicationToSnapshotsRepository

CHANGELOG.md

@@ -1,27 +1,14 @@
 # CHANGELOG
+
 Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased 1.x]
 ### Added
-- Maintainer approval check ([#11378](https://github.com/opensearch-project/OpenSearch/pull/11378))
-
 ### Dependencies
-- Bump `netty` from 4.1.96.Final to 4.1.97.Final ([#9553](https://github.com/opensearch-project/OpenSearch/pull/9553))
-- Bump `netty` from 4.1.100.Final to 4.1.106.Final ([#11294](https://github.com/opensearch-project/OpenSearch/pull/11294), [#11775](https://github.com/opensearch-project/OpenSearch/pull/11775)), [#12034](https://github.com/opensearch-project/OpenSearch/pull/12034))
-- Bump `org.apache.zookeeper:zookeper` from 3.8.0 to 3.8.3 ([#11476](https://github.com/opensearch-project/OpenSearch/pull/11476))
-- Bump `org.apache.avro:avro` from 1.10.2 to 1.11.3 ([#11502](https://github.com/opensearch-project/OpenSearch/pull/11502))
-- Bump `jetty` from 9.4.51.v20230217 to 9.4.52.v20230823 ([#11501](https://github.com/opensearch-project/OpenSearch/pull/11501))
-- Bump `io.projectreactor:reactor-core` from 3.4.23 to 3.4.34 and reactor-netty from 1.0.24 to 1.0.39 ([#11500](https://github.com/opensearch-project/OpenSearch/pull/11500))
-- Bump `logback-core` and `logback-classic` to 1.2.13 ([#11521](https://github.com/opensearch-project/OpenSearch/pull/11521))
-- Bumps `jetty` version from 9.4.52.v20230823 to 9.4.53.v20231009 ([#11539](https://github.com/opensearch-project/OpenSearch/pull/11539))
-- Bump `netty` from 4.1.106.Final to 4.1.108.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372), [#12924](https://github.com/opensearch-project/OpenSearch/pull/12924))
-
 ### Changed
-- Use iterative approach to evaluate Regex.simpleMatch ([#11060](https://github.com/opensearch-project/OpenSearch/pull/11060))
-
 ### Deprecated
 ### Removed
 ### Fixed
 ### Security
 
-[Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/1.3.12...HEAD
+[Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/1.3.15...HEAD

buildSrc/build.gradle

@@ -149,6 +149,20 @@ dependencies {
   integTestImplementation('org.spockframework:spock-core:2.3-groovy-2.5') {
     exclude module: "groovy"
   }
+  constraints {
+    runtimeOnly("org.apache.logging.log4j:log4j-core:${props.getProperty('log4j')}") {
+      because 'log4j CVE'
+    }
+    runtimeOnly("org.jdom:jdom2:${props.getProperty('jdom2')}") {
+      because 'CVE-2021-33813 violation'
+    }
+  }
+}
+
+configurations.all {
+  resolutionStrategy {
+    force "com.google.guava:guava:${props.getProperty('guava')}"
+  }
 }
 
 /*****************************************************************************

buildSrc/version.properties

@@ -14,8 +14,10 @@ icu4j             = 62.1
 supercsv          = 2.4.0
 log4j             = 2.17.1
 slf4j             = 1.6.2
-asm               = 9.5
+jdom2             = 2.0.6.1
+asm               = 9.6
 jettison          = 1.5.4
+asm               = 9.5
 woodstox          = 6.4.0
 kotlin            = 1.7.10
 guava             = 32.0.1-jre

distribution/build.gradle

@@ -420,6 +420,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         }
         from project.jdks."bundled_${platform}_${architecture}"
         exclude "demo/**"
+        exclude "man/ja/**"
         /*
          * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude
          * it from the build.

distribution/tools/keystore-cli/build.gradle

@@ -35,5 +35,5 @@ dependencies {
   compileOnly project(":libs:opensearch-cli")
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.1'
-  testRuntimeOnly 'com.google.guava:guava:32.0.1-jre'
+  testRuntimeOnly "com.google.guava:guava:${versions.guava}"
 }

distribution/tools/plugin-cli/build.gradle

@@ -36,10 +36,10 @@ dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
   api "org.bouncycastle:bcpg-fips:1.0.5.1"
-  api "org.bouncycastle:bc-fips:1.0.2.3"
+  api "org.bouncycastle:bc-fips:1.0.2.4"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly 'com.google.guava:guava:32.0.1-jre'
+  testRuntimeOnly "com.google.guava:guava:${versions.guava}"
 }
 
 tasks.named("dependencyLicenses").configure {

distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1

@@ -0,0 +1 @@
+9008d04fc13da6455e6a792935b93b629757335d

distribution/tools/upgrade-cli/build.gradle

@@ -19,7 +19,7 @@ dependencies {
   implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'
-  testRuntimeOnly 'com.google.guava:guava:32.0.1-jre'
+  testRuntimeOnly "com.google.guava:guava:${versions.guava}"
 }
 
 tasks.named("dependencyLicenses").configure {

modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HeaderVerifierIT.java

@@ -0,0 +1,76 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.netty4;
+
+import org.opensearch.OpenSearchNetty4IntegTestCase;
+import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+import org.opensearch.transport.Netty4MarkedMessagePlugin;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import io.netty.buffer.ByteBufUtil;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.util.ReferenceCounted;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
+public class Netty4HeaderVerifierIT extends OpenSearchNetty4IntegTestCase {
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(Netty4MarkedMessagePlugin.class);
+    }
+
+    public void testThatNettyHttpServerRequestMarksMessageWithHeaderVerifier() throws Exception {
+        HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
+        TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
+        TransportAddress transportAddress = randomFrom(boundAddresses);
+
+        final FullHttpRequest markedRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+        final String expectedMarkedHeaderValue = "Mark with" + randomAlphaOfLength(10);
+        markedRequest.headers().add("marked-message", expectedMarkedHeaderValue);
+
+        final List<FullHttpResponse> responses = new ArrayList<>();
+        try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
+            try {
+                final FullHttpResponse markedResponse = nettyHttpClient.send(transportAddress.address(), markedRequest);
+                responses.add(markedResponse);
+                final String rootResponseContent = new String(ByteBufUtil.getBytes(markedResponse.content()), StandardCharsets.UTF_8);
+                assertThat(rootResponseContent, containsString("opensearch"));
+                assertThat(markedResponse.status().code(), equalTo(200));
+
+                assertThat(Netty4MarkedMessagePlugin.MESSAGE.get().headers().get("marked-message"), equalTo(expectedMarkedHeaderValue));
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
+        }
+
+    }
+
+}

modules/transport-netty4/src/internalClusterTest/java/org/opensearch/transport/Netty4MarkedMessagePlugin.java

@@ -0,0 +1,106 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.PageCacheRecycler;
+import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.common.xcontent.NamedXContentRegistry;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.netty4.Netty4HttpServerTransport;
+import org.opensearch.threadpool.ThreadPool;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.channel.ChannelHandler.Sharable;
+import io.netty.handler.codec.http.DefaultHttpRequest;
+import io.netty.handler.codec.http.HttpMessage;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.util.ReferenceCountUtil;
+
+public class Netty4MarkedMessagePlugin extends Netty4Plugin {
+
+    public static final AtomicReference<HttpMessage> MESSAGE = new AtomicReference<>();
+
+    public class Netty4BlockingHttpServerTransport extends Netty4HttpServerTransport {
+
+        public Netty4BlockingHttpServerTransport(
+            Settings settings,
+            NetworkService networkService,
+            BigArrays bigArrays,
+            ThreadPool threadPool,
+            NamedXContentRegistry xContentRegistry,
+            Dispatcher dispatcher,
+            ClusterSettings clusterSettings,
+            SharedGroupFactory sharedGroupFactory
+        ) {
+            super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, sharedGroupFactory);
+        }
+
+        @Override
+        protected ChannelInboundHandlerAdapter createHeaderVerifier() {
+            return new ExampleBlockingNetty4HeaderVerifier();
+        }
+    }
+
+    @Override
+    public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
+        Settings settings,
+        ThreadPool threadPool,
+        BigArrays bigArrays,
+        PageCacheRecycler pageCacheRecycler,
+        CircuitBreakerService circuitBreakerService,
+        NamedXContentRegistry xContentRegistry,
+        NetworkService networkService,
+        HttpServerTransport.Dispatcher dispatcher,
+        ClusterSettings clusterSettings
+    ) {
+        return Collections.singletonMap(
+            NETTY_HTTP_TRANSPORT_NAME,
+            () -> new Netty4BlockingHttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry,
+                dispatcher,
+                clusterSettings,
+                getSharedGroupFactory(settings)
+            )
+        );
+    }
+
+    /** POC for how an external header verifier would be implemented */
+    @Sharable
+    public class ExampleBlockingNetty4HeaderVerifier extends SimpleChannelInboundHandler<DefaultHttpRequest> {
+
+        @Override
+        public void channelRead0(ChannelHandlerContext ctx, DefaultHttpRequest msg) throws Exception {
+            ReferenceCountUtil.retain(msg);
+            if (isMarked(msg)) {
+                MESSAGE.compareAndSet(null, msg);
+            }
+
+            // Lets the request pass to the next channel handler
+            ctx.fireChannelRead(msg);
+        }
+
+        private boolean isMarked(HttpRequest request) {
+            return request.headers().contains("marked-message");
+        }
+    }
+}

modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java

@@ -314,7 +314,7 @@ public ChannelHandler configureServerChannelHandler() {
         return new HttpChannelHandler(this, handlingSettings);
     }
 
-    static final AttributeKey<Netty4HttpChannel> HTTP_CHANNEL_KEY = AttributeKey.newInstance("es-http-channel");
+    public static final AttributeKey<Netty4HttpChannel> HTTP_CHANNEL_KEY = AttributeKey.newInstance("es-http-channel");
     static final AttributeKey<Netty4HttpServerChannel> HTTP_SERVER_CHANNEL_KEY = AttributeKey.newInstance("es-http-server-channel");
 
     protected static class HttpChannelHandler extends ChannelInitializer<Channel> {
@@ -348,7 +348,8 @@ protected void initChannel(Channel ch) throws Exception {
             );
             decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR);
             ch.pipeline().addLast("decoder", decoder);
-            ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor());
+            ch.pipeline().addLast("header_verifier", transport.createHeaderVerifier());
+            ch.pipeline().addLast("decoder_compress", transport.createDecompressor());
             ch.pipeline().addLast("encoder", new HttpResponseEncoder());
             final HttpObjectAggregator aggregator = new HttpObjectAggregator(handlingSettings.getMaxContentLength());
             aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
@@ -390,4 +391,21 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
             }
         }
     }
+
+    /**
+     * Extension point that allows a NetworkPlugin to extend the netty pipeline and inspect headers after request decoding
+     */
+    protected ChannelInboundHandlerAdapter createHeaderVerifier() {
+        // pass-through
+        return new ChannelInboundHandlerAdapter();
+    }
+
+    /**
+     * Extension point that allows a NetworkPlugin to override the default netty HttpContentDecompressor and supply a custom decompressor.
+     *
+     * Used in instances to conditionally decompress depending on the outcome from header verification
+     */
+    protected ChannelInboundHandlerAdapter createDecompressor() {
+        return new HttpContentDecompressor();
+    }
 }

modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4Plugin.java

@@ -139,7 +139,7 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
         );
     }
 
-    private SharedGroupFactory getSharedGroupFactory(Settings settings) {
+    SharedGroupFactory getSharedGroupFactory(Settings settings) {
         SharedGroupFactory groupFactory = this.groupFactory.get();
         if (groupFactory != null) {
             assert groupFactory.getSettings().equals(settings) : "Different settings than originally provided";

plugins/discovery-ec2/build.gradle

@@ -39,7 +39,7 @@ opensearchplugin {
 }
 
 versions << [
-  'aws': '1.11.749'
+  'aws': '1.12.270'
 ]
 
 dependencies {