Merge remote-tracking branch 'upstream/main' into semver_range_support

CHANGELOG.md

@@ -132,6 +132,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add copy ingest processor ([#11870](https://github.com/opensearch-project/OpenSearch/pull/11870))
 - Introduce new feature flag "WRITEABLE_REMOTE_INDEX" to gate the writeable remote index functionality ([#11717](https://github.com/opensearch-project/OpenSearch/pull/11170))
 - Bump OpenTelemetry from 1.32.0 to 1.34.1 ([#11891](https://github.com/opensearch-project/OpenSearch/pull/11891))
+- Support index level allocation filtering for searchable snapshot index ([#11563](https://github.com/opensearch-project/OpenSearch/pull/11563))
+- Add `org.opensearch.rest.MethodHandlers` and `RestController#getAllHandlers` ([11876](https://github.com/opensearch-project/OpenSearch/pull/11876))
 
 ### Dependencies
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
@@ -175,6 +177,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `Lucene` from 9.8.0 to 9.9.1 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421))
 - Bump `com.networknt:json-schema-validator` from 1.0.86 to 1.1.0 ([#11886](https://github.com/opensearch-project/OpenSearch/pull/11886))
 - Bump `com.google.api:gax-httpjson` from 0.103.1 to 2.39.0 ([#11794](https://github.com/opensearch-project/OpenSearch/pull/11794))
+- Bump `com.google.oauth-client:google-oauth-client` from 1.34.1 to 1.35.0 ([#11960](https://github.com/opensearch-project/OpenSearch/pull/11960))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -205,6 +208,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use slice_size == shard_size heuristic in terms aggs for concurrent segment search and properly calculate the doc_count_error ([#11732](https://github.com/opensearch-project/OpenSearch/pull/11732))
 - Added Support for dynamically adding SearchRequestOperationsListeners with SearchRequestOperationsCompositeListenerFactory ([#11526](https://github.com/opensearch-project/OpenSearch/pull/11526))
 - Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
+- Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877))
 
 ### Deprecated
 
@@ -234,6 +238,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix issue when calling Delete PIT endpoint and no PITs exist ([#11711](https://github.com/opensearch-project/OpenSearch/pull/11711))
 - Fix tracing context propagation for local transport instrumentation ([#11490](https://github.com/opensearch-project/OpenSearch/pull/11490))
 - Fix parsing of single line comments in `lang-painless` ([#11815](https://github.com/opensearch-project/OpenSearch/issues/11815))
+- Fix memory leak issue in ReorganizingLongHash ([#11953](https://github.com/opensearch-project/OpenSearch/issues/11953))
 
 ### Security
 

client/sniffer/src/main/java/org/opensearch/client/sniff/Sniffer.java

@@ -305,6 +305,7 @@ public void shutdown() {
         }
     }
 
+    @SuppressWarnings("removal")
     static class SnifferThreadFactory implements ThreadFactory {
         private final AtomicInteger threadNumber = new AtomicInteger(1);
         private final String namePrefix;

distribution/archives/integ-test-zip/src/test/java/org/opensearch/test/rest/JsonLogsFormatAndParseIT.java

@@ -51,6 +51,7 @@ protected Matcher<String> nodeNameMatcher() {
         return is("integTest-0");
     }
 
+    @SuppressWarnings("removal")
     @Override
     protected BufferedReader openReader(Path logFile) {
         assumeFalse("Skipping test because it is being run against an external cluster.",

libs/common/src/main/java20/org/opensearch/common/round/RoundableFactory.java

@@ -10,8 +10,6 @@
 
 import org.opensearch.common.annotation.InternalApi;
 
-import jdk.incubator.vector.LongVector;
-
 /**
  * Factory class to create and return the fastest implementation of {@link Roundable}.
  *
@@ -34,10 +32,30 @@ public final class RoundableFactory {
      */
     private static final boolean USE_BTREE_SEARCHER;
 
+    /**
+     * This class is initialized only when:
+     *  - JDK-20+
+     *  - jdk.incubator.vector.LongVector is available (--add-modules=jdk.incubator.vector is passed)
+     */
+    private static final class VectorCheck {
+        final static int SPECIES_PREFERRED = jdk.incubator.vector.LongVector.SPECIES_PREFERRED.length();
+    }
+
     static {
         String simdRoundingFeatureFlag = System.getProperty("opensearch.experimental.feature.simd.rounding.enabled");
-        USE_BTREE_SEARCHER = "forced".equalsIgnoreCase(simdRoundingFeatureFlag)
-            || (LongVector.SPECIES_PREFERRED.length() >= 4 && "true".equalsIgnoreCase(simdRoundingFeatureFlag));
+        boolean useBtreeSearcher = false;
+
+        try {
+            final Class<?> incubator = Class.forName("jdk.incubator.vector.LongVector");
+
+            useBtreeSearcher = "forced".equalsIgnoreCase(simdRoundingFeatureFlag)
+                || (VectorCheck.SPECIES_PREFERRED >= 4 && "true".equalsIgnoreCase(simdRoundingFeatureFlag));
+
+        } catch (final ClassNotFoundException ex) {
+            /* do not use BtreeSearcher */
+        }
+
+        USE_BTREE_SEARCHER = useBtreeSearcher;
     }
 
     private RoundableFactory() {}

libs/common/src/test/java/org/opensearch/common/annotation/processor/CompilerSupport.java

@@ -43,6 +43,7 @@ default CompilerResult compile(String name, String... names) {
         return compileWithPackage(ApiAnnotationProcessorTests.class.getPackageName(), name, names);
     }
 
+    @SuppressWarnings("removal")
     default CompilerResult compileWithPackage(String pck, String name, String... names) {
         final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
         final DiagnosticCollector<JavaFileObject> collector = new DiagnosticCollector<>();

libs/nio/src/main/java/org/opensearch/nio/ServerChannelContext.java

@@ -129,6 +129,7 @@ private void configureSocket(ServerSocket socket) throws IOException {
         socket.setReuseAddress(config.tcpReuseAddress());
     }
 
+    @SuppressWarnings("removal")
     protected static SocketChannel accept(ServerSocketChannel serverSocketChannel) throws IOException {
         try {
             assert serverSocketChannel.isBlocking() == false;

libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java

@@ -388,6 +388,7 @@ private void configureSocket(Socket socket, boolean isConnectComplete) throws IO
         }
     }
 
+    @SuppressWarnings("removal")
     private static void connect(SocketChannel socketChannel, InetSocketAddress remoteAddress) throws IOException {
         try {
             AccessController.doPrivileged((PrivilegedExceptionAction<Boolean>) () -> socketChannel.connect(remoteAddress));

libs/plugin-classloader/src/main/java/org/opensearch/plugins/ExtendedPluginsClassLoader.java

@@ -65,6 +65,7 @@ protected Class<?> findClass(String name) throws ClassNotFoundException {
     /**
      * Return a new classloader across the parent and extended loaders.
      */
+    @SuppressWarnings("removal")
     public static ExtendedPluginsClassLoader create(ClassLoader parent, List<ClassLoader> extendedLoaders) {
         return AccessController.doPrivileged(
             (PrivilegedAction<ExtendedPluginsClassLoader>) () -> new ExtendedPluginsClassLoader(parent, extendedLoaders)

libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecureSM.java

@@ -76,6 +76,7 @@
  * @see <a href="http://cs.oswego.edu/pipermail/concurrency-interest/2009-August/006508.html">
  *         http://cs.oswego.edu/pipermail/concurrency-interest/2009-August/006508.html</a>
  */
+@SuppressWarnings("removal")
 public class SecureSM extends SecurityManager {
 
     private final String[] classesThatCanExit;

libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java

@@ -18,6 +18,7 @@
 import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory;
 import java.util.concurrent.ForkJoinWorkerThread;
 
+@SuppressWarnings("removal")
 public class SecuredForkJoinWorkerThreadFactory implements ForkJoinWorkerThreadFactory {
     static AccessControlContext contextWithPermissions(Permission... perms) {
         Permissions permissions = new Permissions();

libs/secure-sm/src/test/java/org/opensearch/secure_sm/SecureSMTests.java

@@ -41,6 +41,7 @@
 import junit.framework.TestCase;
 
 /** Simple tests for SecureSM */
+@SuppressWarnings("removal")
 public class SecureSMTests extends TestCase {
     static {
         // install a mock security policy:

modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java

@@ -39,7 +39,7 @@
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.query.Operator;
 import org.opensearch.plugins.Plugin;
-import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -49,10 +49,10 @@
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 
-public class QueryStringWithAnalyzersIT extends ParameterizedOpenSearchIntegTestCase {
+public class QueryStringWithAnalyzersIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
 
-    public QueryStringWithAnalyzersIT(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public QueryStringWithAnalyzersIT(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @ParametersFactory

modules/analysis-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java

@@ -44,7 +44,7 @@
 import org.opensearch.plugins.Plugin;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -68,10 +68,10 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.startsWith;
 
-public class HighlighterWithAnalyzersTests extends ParameterizedOpenSearchIntegTestCase {
+public class HighlighterWithAnalyzersTests extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
 
-    public HighlighterWithAnalyzersTests(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public HighlighterWithAnalyzersTests(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @ParametersFactory

modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java

@@ -16,7 +16,7 @@
 import org.opensearch.geometry.utils.WellKnownText;
 import org.opensearch.index.mapper.GeoShapeFieldMapper;
 import org.opensearch.plugins.Plugin;
-import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.TestGeoShapeFieldMapperPlugin;
 
 import java.util.Arrays;
@@ -29,14 +29,14 @@
  * This is the base class for all the Geo related integration tests. Use this class to add the features and settings
  * for the test cluster on which integration tests are running.
  */
-public abstract class GeoModulePluginIntegTestCase extends ParameterizedOpenSearchIntegTestCase {
+public abstract class GeoModulePluginIntegTestCase extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
 
     protected static final double GEOHASH_TOLERANCE = 1E-5D;
 
     protected static final WellKnownText WKT = new WellKnownText(true, new StandardValidator(true));
 
-    public GeoModulePluginIntegTestCase(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public GeoModulePluginIntegTestCase(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @ParametersFactory

modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java

@@ -44,8 +44,8 @@ public class MissingValueIT extends GeoModulePluginIntegTestCase {
     private GeoPoint bottomRight;
     private GeoPoint topLeft;
 
-    public MissingValueIT(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public MissingValueIT(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @Override

modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/GeoIpProcessor.java

@@ -218,6 +218,7 @@ Set<Property> getProperties() {
         return properties;
     }
 
+    @SuppressWarnings("removal")
     private Map<String, Object> retrieveCityGeoData(InetAddress ipAddress) {
         SpecialPermission.check();
         CityResponse response = AccessController.doPrivileged(
@@ -305,6 +306,7 @@ private Map<String, Object> retrieveCityGeoData(InetAddress ipAddress) {
         return geoData;
     }
 
+    @SuppressWarnings("removal")
     private Map<String, Object> retrieveCountryGeoData(InetAddress ipAddress) {
         SpecialPermission.check();
         CountryResponse response = AccessController.doPrivileged(
@@ -351,6 +353,7 @@ private Map<String, Object> retrieveCountryGeoData(InetAddress ipAddress) {
         return geoData;
     }
 
+    @SuppressWarnings("removal")
     private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) {
         SpecialPermission.check();
         AsnResponse response = AccessController.doPrivileged(

modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java

@@ -57,7 +57,7 @@
 import org.opensearch.search.aggregations.pipeline.SimpleValue;
 import org.opensearch.search.sort.SortBuilders;
 import org.opensearch.search.sort.SortOrder;
-import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.hamcrest.OpenSearchAssertions;
 
 import java.util.Arrays;
@@ -80,10 +80,10 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 // TODO: please convert to unit tests!
-public class MoreExpressionIT extends ParameterizedOpenSearchIntegTestCase {
+public class MoreExpressionIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
 
-    public MoreExpressionIT(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public MoreExpressionIT(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @ParametersFactory

modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java

@@ -43,7 +43,7 @@
 import org.opensearch.script.ScriptType;
 import org.opensearch.search.aggregations.AggregationBuilders;
 import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -54,10 +54,10 @@
 import static org.hamcrest.Matchers.containsString;
 
 //TODO: please convert to unit tests!
-public class StoredExpressionIT extends ParameterizedOpenSearchIntegTestCase {
+public class StoredExpressionIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
 
-    public StoredExpressionIT(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public StoredExpressionIT(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @ParametersFactory

modules/lang-expression/src/main/java/org/opensearch/script/expression/ExpressionScriptEngine.java

@@ -170,6 +170,7 @@ public String getType() {
         return NAME;
     }
 
+    @SuppressWarnings("removal")
     @Override
     public <T> T compile(String scriptName, String scriptSource, ScriptContext<T> context, Map<String, String> params) {
         // classloader created here

modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/MultiSearchTemplateIT.java

@@ -41,7 +41,7 @@
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.script.ScriptType;
-import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -58,10 +58,10 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.core.Is.is;
 
-public class MultiSearchTemplateIT extends ParameterizedOpenSearchIntegTestCase {
+public class MultiSearchTemplateIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
 
-    public MultiSearchTemplateIT(Settings dynamicSettings) {
-        super(dynamicSettings);
+    public MultiSearchTemplateIT(Settings staticSettings) {
+        super(staticSettings);
     }
 
     @ParametersFactory

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MustacheScriptEngine.java

@@ -128,6 +128,7 @@ private class MustacheExecutableScript extends TemplateScript {
             this.params = params;
         }
 
+        @SuppressWarnings("removal")
         @Override
         public String execute() {
             final StringWriter writer = new StringWriter();

modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java

@@ -513,6 +513,7 @@ public static Allowlist loadFromResourceFiles(Class<?> resource, Map<String, All
             }
         }
 
+        @SuppressWarnings("removal")
         ClassLoader loader = AccessController.doPrivileged((PrivilegedAction<ClassLoader>) resource::getClassLoader);
 
         return new Allowlist(loader, allowlistClasses, allowlistStatics, allowlistClassBindings, Collections.emptyList());

modules/lang-painless/src/main/java/org/opensearch/painless/LambdaBootstrap.java

@@ -501,6 +501,7 @@ private static void endLambdaClass(ClassWriter cw) {
      * Defines the {@link Class} for the lambda class using the same {@link Compiler.Loader}
      * that originally defined the class for the Painless script.
      */
+    @SuppressWarnings("removal")
     private static Class<?> createLambdaClass(Compiler.Loader loader, ClassWriter cw, Type lambdaClassType) {
 
         byte[] classBytes = cw.toByteArray();

modules/lang-painless/src/main/java/org/opensearch/painless/PainlessScriptEngine.java

@@ -67,6 +67,7 @@
 /**
  * Implementation of a ScriptEngine for the Painless language.
  */
+@SuppressWarnings("removal")
 public final class PainlessScriptEngine implements ScriptEngine {
 
     /**

modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java

@@ -2189,6 +2189,7 @@ private void generateBridgeMethod(PainlessClassBuilder painlessClassBuilder, Pai
             bridgeClassWriter.visitEnd();
 
             try {
+                @SuppressWarnings("removal")
                 BridgeLoader bridgeLoader = AccessController.doPrivileged(new PrivilegedAction<BridgeLoader>() {
                     @Override
                     public BridgeLoader run() {

modules/lang-painless/src/test/java/org/opensearch/painless/DocFieldsPhaseTests.java

@@ -48,6 +48,7 @@
 public class DocFieldsPhaseTests extends ScriptTestCase {
     PainlessLookup lookup = PainlessLookupBuilder.buildFromAllowlists(Allowlist.BASE_ALLOWLISTS);
 
+    @SuppressWarnings("removal")
     ScriptScope compile(String script) {
         Compiler compiler = new Compiler(
             MockDocTestScript.CONTEXT.instanceClazz,