Merge branch 'main' into keyword_terms

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

.ci/java-versions.properties

@@ -13,7 +13,8 @@
 # build and test OpenSearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-OPENSEARCH_BUILD_JAVA=openjdk11
+# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+OPENSEARCH_BUILD_JAVA=openjdk17
 OPENSEARCH_RUNTIME_JAVA=java11
 GRADLE_TASK=build
 GRADLE_EXTRA_ARGS=

.github/CODEOWNERS

@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jainankitk @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
 /modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 /modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah

.github/workflows/assemble.yml

@@ -16,6 +16,17 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
+      - name: Set up JDK 17
+        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+        if: matrix.java == 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+      - name: Set JAVA${{ matrix.java }}_HOME
+        shell: bash
+        run: |
+          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
@@ -30,10 +41,11 @@ jobs:
           # Report success even if previous step failed (Docker on MacOS runner is very unstable)
           exit 0;
       - name: Run Gradle (assemble)
+        shell: bash
         if: runner.os != 'macos'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}

.github/workflows/dependabot_pr.yml

@@ -22,6 +22,13 @@ jobs:
         with:
           token: ${{ steps.github_app_token.outputs.token }}
 
+      # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+
       - name: Update Gradle SHAs
         run: |
           ./gradlew updateSHAs

.github/workflows/precommit.yml

@@ -17,6 +17,18 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
+      - name: Set up JDK 17
+        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
+        if: matrix.java == 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+      - name: Set JAVA${{ matrix.java }}_HOME
+        shell: bash
+        run: |
+          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Run Gradle (precommit)
+        shell: bash
         run: |
-          ./gradlew javadoc precommit --parallel
+          ./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}

CHANGELOG.md

@@ -17,7 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072))
 - Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711))
 - Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054))
+- [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709))
 - [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
+- Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774))
 - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153))
 
 ### Dependencies
@@ -29,13 +31,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110))
 - Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104))
 - Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119))
-- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.8 and `org.bouncycastle:bc-fips` from 1.0.2.5 to 2.0.0 in /distribution/tools/plugin-cli ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103))
+- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.9 ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103), [#15299](https://github.com/opensearch-project/OpenSearch/pull/15299))
 - Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111))
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.5 to 1.1.10.6 ([#15207](https://github.com/opensearch-project/OpenSearch/pull/15207))
 - Bump `com.azure:azure-xml` from 1.0.0 to 1.1.0 ([#15206](https://github.com/opensearch-project/OpenSearch/pull/15206))
+- Bump `reactor` from 3.5.19 to 3.5.20 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262))
+- Bump `reactor-netty` from 1.1.21 to 1.1.22 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262))
+- Bump `org.apache.kerby:kerb-admin` from 2.0.3 to 2.1.0 ([#15301](https://github.com/opensearch-project/OpenSearch/pull/15301))
+- Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300))
+- Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297))
+- Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298))
 
 ### Changed
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
+- Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371))
 - Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238))
 - Updated `termsQuery` in `KeywordField` to set custom rewrite_override before executing query
 
@@ -50,6 +59,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080))
 - Fix array_index_out_of_bounds_exception when indexing documents with field name containing only dot ([#15126](https://github.com/opensearch-project/OpenSearch/pull/15126))
 - Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620))
+- Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194))
+- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233))
 
 ### Security
 

MAINTAINERS.md

@@ -9,6 +9,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Anas Alkouz              | [anasalkouz](https://github.com/anasalkouz)             | Amazon      |
 | Andrew Ross              | [andrross](https://github.com/andrross)                 | Amazon      |
 | Andriy Redko             | [reta](https://github.com/reta)                         | Aiven       |
+| Ankit Jain               | [jainankitk](https://github.com/jainankitk)             | Amazon      |
 | Ashish Singh             | [ashking94](https://github.com/ashking94)               | Amazon      |
 | Bukhtawar Khan           | [Bukhtawar](https://github.com/Bukhtawar)               | Amazon      |
 | Charlotte Henkle         | [CEHENKLE](https://github.com/CEHENKLE)                 | Amazon      |

buildSrc/src/main/resources/minimumCompilerVersion

@@ -1 +1 @@
-14
+17

buildSrc/version.properties

@@ -33,8 +33,8 @@ netty             = 4.1.112.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.21
-reactor           = 3.5.19
+reactor_netty     = 1.1.22
+reactor           = 3.5.20
 
 # client dependencies
 httpclient5       = 5.2.3

client/rest/licenses/reactor-core-3.5.20.jar.sha1

@@ -0,0 +1 @@
+1fc0f91e2b93778a974339d2c24363d7f34f90b4

distribution/tools/plugin-cli/build.gradle

@@ -37,7 +37,7 @@ base {
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
-  api "org.bouncycastle:bcpg-fips:2.0.8"
+  api "org.bouncycastle:bcpg-fips:2.0.9"
   api "org.bouncycastle:bc-fips:2.0.0"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.3.0'

distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.9.jar.sha1

@@ -0,0 +1 @@
+f69719ef8dbf34d5f906ce480496446b2fd2ae27

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
+distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703

modules/analysis-common/src/main/java/org/opensearch/analysis/common/MinHashTokenFilterFactory.java

@@ -65,10 +65,10 @@ private Map<String, String> convertSettings(Settings settings) {
         if (settings.hasValue("hash_count")) {
             settingMap.put("hashCount", settings.get("hash_count"));
         }
-        if (settings.hasValue("bucketCount")) {
+        if (settings.hasValue("bucket_count")) {
             settingMap.put("bucketCount", settings.get("bucket_count"));
         }
-        if (settings.hasValue("hashSetSize")) {
+        if (settings.hasValue("hash_set_size")) {
             settingMap.put("hashSetSize", settings.get("hash_set_size"));
         }
         if (settings.hasValue("with_rotation")) {

modules/analysis-common/src/test/java/org/opensearch/analysis/common/MinHashFilterFactoryTests.java

@@ -50,14 +50,10 @@ public void testDefault() throws IOException {
         int default_bucket_size = 512;
         int default_hash_set_size = 1;
         Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
-        OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
-            settings,
-            new CommonAnalysisModulePlugin()
-        );
+        OpenSearchTestCase.TestAnalysis analysis = getTestAnalysisFromSettings(settings);
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("min_hash");
         String source = "the quick brown fox";
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
+        Tokenizer tokenizer = getTokenizer(source);
 
         // with_rotation is true by default, and hash_set_size is 1, so even though the source doesn't
         // have enough tokens to fill all the buckets, we still expect 512 tokens.
@@ -73,17 +69,63 @@ public void testSettings() throws IOException {
             .put("index.analysis.filter.test_min_hash.with_rotation", false)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
-            settings,
-            new CommonAnalysisModulePlugin()
-        );
+        OpenSearchTestCase.TestAnalysis analysis = getTestAnalysisFromSettings(settings);
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("test_min_hash");
         String source = "sushi";
-        Tokenizer tokenizer = new WhitespaceTokenizer();
-        tokenizer.setReader(new StringReader(source));
+        Tokenizer tokenizer = getTokenizer(source);
 
         // despite the fact that bucket_count is 2 and hash_set_size is 1,
         // because with_rotation is false, we only expect 1 token here.
         assertStreamHasNumberOfTokens(tokenFilter.create(tokenizer), 1);
     }
+
+    public void testBucketCountSetting() throws IOException {
+        // Correct case with "bucket_count"
+        Settings settingsWithBucketCount = Settings.builder()
+            .put("index.analysis.filter.test_min_hash.type", "min_hash")
+            .put("index.analysis.filter.test_min_hash.hash_count", "1")
+            .put("index.analysis.filter.test_min_hash.bucket_count", "3")
+            .put("index.analysis.filter.test_min_hash.hash_set_size", "1")
+            .put("index.analysis.filter.test_min_hash.with_rotation", false)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+
+        OpenSearchTestCase.TestAnalysis analysisWithBucketCount = getTestAnalysisFromSettings(settingsWithBucketCount);
+
+        TokenFilterFactory tokenFilterWithBucketCount = analysisWithBucketCount.tokenFilter.get("test_min_hash");
+        String sourceWithBucketCount = "salmon avocado roll uramaki";
+        Tokenizer tokenizerWithBucketCount = getTokenizer(sourceWithBucketCount);
+        // Expect 3 tokens due to bucket_count being set to 3
+        assertStreamHasNumberOfTokens(tokenFilterWithBucketCount.create(tokenizerWithBucketCount), 3);
+    }
+
+    public void testHashSetSizeSetting() throws IOException {
+        // Correct case with "hash_set_size"
+        Settings settingsWithHashSetSize = Settings.builder()
+            .put("index.analysis.filter.test_min_hash.type", "min_hash")
+            .put("index.analysis.filter.test_min_hash.hash_count", "1")
+            .put("index.analysis.filter.test_min_hash.bucket_count", "1")
+            .put("index.analysis.filter.test_min_hash.hash_set_size", "2")
+            .put("index.analysis.filter.test_min_hash.with_rotation", false)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+
+        OpenSearchTestCase.TestAnalysis analysisWithHashSetSize = getTestAnalysisFromSettings(settingsWithHashSetSize);
+
+        TokenFilterFactory tokenFilterWithHashSetSize = analysisWithHashSetSize.tokenFilter.get("test_min_hash");
+        String sourceWithHashSetSize = "salmon avocado roll uramaki";
+        Tokenizer tokenizerWithHashSetSize = getTokenizer(sourceWithHashSetSize);
+        // Expect 2 tokens due to hash_set_size being set to 2 and bucket_count being 1
+        assertStreamHasNumberOfTokens(tokenFilterWithHashSetSize.create(tokenizerWithHashSetSize), 2);
+    }
+
+    private static OpenSearchTestCase.TestAnalysis getTestAnalysisFromSettings(Settings settingsWithBucketCount) throws IOException {
+        return AnalysisTestsHelper.createTestAnalysisFromSettings(settingsWithBucketCount, new CommonAnalysisModulePlugin());
+    }
+
+    private static Tokenizer getTokenizer(String sourceWithBucketCount) {
+        Tokenizer tokenizerWithBucketCount = new WhitespaceTokenizer();
+        tokenizerWithBucketCount.setReader(new StringReader(sourceWithBucketCount));
+        return tokenizerWithBucketCount;
+    }
 }

plugins/repository-azure/build.gradle

@@ -48,7 +48,7 @@ dependencies {
   api 'com.azure:azure-json:1.1.0'
   api 'com.azure:azure-xml:1.1.0'
   api 'com.azure:azure-storage-common:12.25.1'
-  api 'com.azure:azure-core-http-netty:1.15.1'
+  api 'com.azure:azure-core-http-netty:1.15.3'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"

plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1

@@ -0,0 +1 @@
+03b5bd5f5c16eea71f130119dbfb1fe5239f806a

plugins/repository-azure/licenses/reactor-netty-core-1.1.22.jar.sha1

@@ -0,0 +1 @@
+08356b59b29f86e7142c9daca0434653a64ae64b

plugins/repository-azure/licenses/reactor-netty-http-1.1.22.jar.sha1

@@ -0,0 +1 @@
+2faf64b3822b0512f15d72a325e2826eb8564413

plugins/repository-hdfs/build.gradle

@@ -70,7 +70,7 @@ dependencies {
   api 'com.google.code.gson:gson:2.11.0'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api 'commons-cli:commons-cli:1.8.0'
+  api 'commons-cli:commons-cli:1.9.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"

plugins/repository-hdfs/licenses/commons-cli-1.9.0.jar.sha1

@@ -0,0 +1 @@
+e1cdfa8bf40ccbb7440b2d1232f9f45bb20a1844

plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.22.jar.sha1

@@ -0,0 +1 @@
+08356b59b29f86e7142c9daca0434653a64ae64b

plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.22.jar.sha1

@@ -0,0 +1 @@
+2faf64b3822b0512f15d72a325e2826eb8564413

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java

@@ -18,8 +18,11 @@
 import org.opensearch.common.settings.SettingsFilter;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.plugin.wlm.action.CreateQueryGroupAction;
+import org.opensearch.plugin.wlm.action.GetQueryGroupAction;
 import org.opensearch.plugin.wlm.action.TransportCreateQueryGroupAction;
+import org.opensearch.plugin.wlm.action.TransportGetQueryGroupAction;
 import org.opensearch.plugin.wlm.rest.RestCreateQueryGroupAction;
+import org.opensearch.plugin.wlm.rest.RestGetQueryGroupAction;
 import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService;
 import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.plugins.Plugin;
@@ -41,7 +44,10 @@ public WorkloadManagementPlugin() {}
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return List.of(new ActionPlugin.ActionHandler<>(CreateQueryGroupAction.INSTANCE, TransportCreateQueryGroupAction.class));
+        return List.of(
+            new ActionPlugin.ActionHandler<>(CreateQueryGroupAction.INSTANCE, TransportCreateQueryGroupAction.class),
+            new ActionPlugin.ActionHandler<>(GetQueryGroupAction.INSTANCE, TransportGetQueryGroupAction.class)
+        );
     }
 
     @Override
@@ -54,7 +60,7 @@ public List<RestHandler> getRestHandlers(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<DiscoveryNodes> nodesInCluster
     ) {
-        return List.of(new RestCreateQueryGroupAction());
+        return List.of(new RestCreateQueryGroupAction(), new RestGetQueryGroupAction());
     }
 
     @Override