Merge branch 'main' into maintainer-approved-check

Signed-off-by: Peter Nied <[email protected]>

.github/workflows/lucene-snapshots.yml

@@ -7,7 +7,8 @@ on:
     # Inputs the workflow accepts.
     inputs:
       ref:
-        description:
+        description: 'Lucene ref in github.com/apache/lucene'
+        type: string
         required: false
         default: 'main'
 
@@ -21,40 +22,60 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: '17'
-          distribution: 'adopt'
-
-      - name: Checkout Lucene
+      - name: Checkout Lucene ref:${{ github.event.inputs.ref }}
         uses: actions/checkout@v4
         with:
           repository: 'apache/lucene'
-          path: lucene
           ref: ${{ github.event.inputs.ref }}
 
-      - name: Set hash
-        working-directory: ./lucene
+      - name: Get Java Min Version and Lucene Revision from Lucene Repository
         run: |
-          echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        id: version
+          java_version=`cat build.gradle | grep minJavaVersion | head -1 | grep -Eo '_[0-9]+$' | tr -d '_'`
+          echo "JAVA_VERSION=$java_version" >> $GITHUB_ENV
+          echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Setup JDK ${{ env.JAVA_VERSION }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: 'temurin'
 
       - name: Initialize gradle settings
-        working-directory: ./lucene
         run: ./gradlew localSettings
 
       - name: Publish Lucene to local maven repo.
-        working-directory: ./lucene
-        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
+        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ env.REVISION }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_SECRET_ROLE }}
+          aws-region: us-east-1
+
+      - name: Get S3 Bucket
+        id: get_s3_bucket
+        run: |
+          lucene_snapshots_bucket=`aws secretsmanager get-secret-value --secret-id jenkins-artifact-bucket-name --query SecretString --output text`
+          echo "::add-mask::$lucene_snapshots_bucket"
+          echo "LUCENE_SNAPSHOTS_BUCKET=$lucene_snapshots_bucket" >> $GITHUB_OUTPUT
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+
+      - name: Copy files to S3 with the aws CLI (New)
+        run: |
+          aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ steps.get_s3_bucket.outputs.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
           aws-region: us-west-2
 
-      - name: Copy files to S3 with the aws CLI.
+        # We will remove this step once all the lucene snapshots old links are updated with the new one
+      - name: Copy files to S3 with the aws CLI (Old)
         run: |
           aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ secrets.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress

CHANGELOG.md

@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
 - Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
 - Make number of segment metadata files in remote segment store configurable ([#11329](https://github.com/opensearch-project/OpenSearch/pull/11329))
+- [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
 - Maintainer approval check ([#11378](https://github.com/opensearch-project/OpenSearch/pull/11378))
 
 ### Dependencies
@@ -121,7 +122,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
 - Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
-- Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
+- Bump `commons-io:commons-io` from 2.13.0 to 2.15.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294), [#11001](https://github.com/opensearch-project/OpenSearch/pull/11001), [#11002](https://github.com/opensearch-project/OpenSearch/pull/11002))
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
@@ -153,6 +154,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 - Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
 - Improve boolean parsing performance ([#11308](https://github.com/opensearch-project/OpenSearch/pull/11308))
+- Change error message when per shard document limit is breached ([#11312](https://github.com/opensearch-project/OpenSearch/pull/11312))
 
 ### Deprecated
 
@@ -169,6 +171,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Delegating CachingWeightWrapper#count to internal weight object ([#10543](https://github.com/opensearch-project/OpenSearch/pull/10543))
 - Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934))
 - Fix for stuck update action in a bulk with `retry_on_conflict` property ([#11152](https://github.com/opensearch-project/OpenSearch/issues/11152))
+- Remove shadowJar from `lang-painless` module publication ([#11369](https://github.com/opensearch-project/OpenSearch/issues/11369))
 
 ### Security
 

modules/lang-painless/build.gradle

@@ -33,7 +33,6 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin
 
 apply plugin: 'opensearch.validate-rest-spec'
 apply plugin: 'opensearch.yaml-rest-test'
-apply plugin: 'com.github.johnrengelman.shadow'
 
 opensearchplugin {
   description 'An easy, safe and fast scripting language for OpenSearch'
@@ -62,30 +61,6 @@ dependencies {
   api project('spi')
 }
 
-test {
-  doFirst {
-    test.classpath -= project.files(project.tasks.named('shadowJar'))
-    test.classpath -= project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME)
-    test.classpath += project.extensions.getByType(SourceSetContainer).getByName(SourceSet.MAIN_SOURCE_SET_NAME).runtimeClasspath
-  }
-}
-
-shadowJar {
-  archiveClassifier.set('')
-  relocate 'org.objectweb', 'org.opensearch.repackage.org.objectweb'
-  dependencies {
-    include(dependency("org.ow2.asm:asm:${versions.asm}"))
-    include(dependency("org.ow2.asm:asm-util:${versions.asm}"))
-    include(dependency("org.ow2.asm:asm-tree:${versions.asm}"))
-    include(dependency("org.ow2.asm:asm-commons:${versions.asm}"))
-    include(dependency("org.ow2.asm:asm-analysis:${versions.asm}"))
-  }
-}
-
-tasks.validateNebulaPom.dependsOn tasks.generatePomFileForShadowPublication
-tasks.validateShadowPom.dependsOn tasks.generatePomFileForNebulaPublication
-tasks.withType(AbstractPublishToMaven)*.dependsOn "generatePomFileForShadowPublication", "generatePomFileForNebulaPublication"
-
 tasks.named("dependencyLicenses").configure {
   mapping from: /asm-.*/, to: 'asm'
 }

plugins/discovery-azure-classic/build.gradle

@@ -53,7 +53,7 @@ dependencies {
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "commons-lang:commons-lang:2.6"
-  api "commons-io:commons-io:2.13.0"
+  api "commons-io:commons-io:2.15.0"
   api 'javax.mail:mail:1.4.7'
   api 'javax.inject:javax.inject:1'
   api "com.sun.jersey:jersey-client:${versions.jersey}"

plugins/discovery-azure-classic/licenses/commons-io-2.15.0.jar.sha1

@@ -0,0 +1 @@
+5c3c2db10f6f797430a7f9c696b4d1273768c924

plugins/ingest-attachment/build.gradle

@@ -57,7 +57,7 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   // Other dependencies
   api 'org.tukaani:xz:1.9'
-  api 'commons-io:commons-io:2.14.0'
+  api 'commons-io:commons-io:2.15.0'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
   // character set detection

plugins/ingest-attachment/licenses/commons-io-2.15.0.jar.sha1

@@ -0,0 +1 @@
+5c3c2db10f6f797430a7f9c696b4d1273768c924

server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java

@@ -35,23 +35,27 @@
 import org.opensearch.action.admin.indices.get.GetIndexRequest;
 import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.index.Index;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
 import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING;
 
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class ClusterIndexRefreshIntervalIT extends OpenSearchIntegTestCase {
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 2)
+public class ClusterIndexRefreshIntervalIT extends AbstractSnapshotIntegTestCase {
 
     public static final String INDEX_NAME = "test-index";
 
@@ -69,9 +73,39 @@ public void setUp() throws Exception {
         internalCluster().startClusterManagerOnlyNode();
     }
 
+    static void putIndexTemplate(String refreshInterval) {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest("my-template"); // <1>
+        request.patterns(Arrays.asList("pattern-1", "log-*")); // <2>
+
+        request.settings(
+            Settings.builder() // <1>
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 1)
+                .put("index.refresh_interval", refreshInterval)
+        );
+        assertTrue(client().admin().indices().putTemplate(request).actionGet().isAcknowledged());
+    }
+
+    public void testIndexTemplateCreationSucceedsWhenNoMinimumRefreshInterval() throws ExecutionException, InterruptedException {
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
+        putIndexTemplate("2s");
+
+        // Test index creation using template with valid refresh interval
+        String indexName = "log-myindex-1";
+        createIndex(indexName);
+        ensureYellowAndNoInitializingShards(indexName);
+        ensureGreen(indexName);
+        GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
+        String uuid = getIndexResponse.getSettings().get(indexName).get(IndexMetadata.SETTING_INDEX_UUID);
+        IndexService indexService = indicesService.indexService(new Index(indexName, uuid));
+        assertEquals(TimeValue.timeValueSeconds(2), indexService.getRefreshTaskInterval());
+    }
+
     public void testDefaultRefreshIntervalWithUpdateClusterAndIndexSettings() throws Exception {
         String clusterManagerName = internalCluster().getClusterManagerName();
-        List<String> dataNodes = internalCluster().startDataOnlyNodes(2);
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         createIndex(INDEX_NAME);
         ensureYellowAndNoInitializingShards(INDEX_NAME);
         ensureGreen(INDEX_NAME);
@@ -90,7 +124,7 @@ public void testDefaultRefreshIntervalWithUpdateClusterAndIndexSettings() throws
             .get();
         assertEquals(refreshInterval, indexService.getRefreshTaskInterval());
 
-        // Update of cluster.minimum.index.refresh_interval setting to value less than refreshInterval above will fail
+        // Update of cluster.minimum.index.refresh_interval setting to value more than default refreshInterval above will fail
         TimeValue invalidMinimumRefreshInterval = TimeValue.timeValueMillis(refreshInterval.millis() + randomIntBetween(1, 1000));
         IllegalArgumentException exceptionDuringMinUpdate = assertThrows(
             IllegalArgumentException.class,
@@ -205,7 +239,7 @@ public void testRefreshIntervalDisabled() throws ExecutionException, Interrupted
             .getAsTime(IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE);
         boolean createIndexSuccess = clusterMinimumRefreshInterval.equals(TimeValue.MINUS_ONE);
         String clusterManagerName = internalCluster().getClusterManagerName();
-        List<String> dataNodes = internalCluster().startDataOnlyNodes(2);
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         Settings settings = Settings.builder()
             .put(indexSettings())
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), IndexSettings.MINIMUM_REFRESH_INTERVAL)
@@ -236,7 +270,7 @@ protected TimeValue getMinRefreshIntervalForRefreshDisabled() {
 
     public void testInvalidRefreshInterval() {
         String invalidRefreshInterval = "-10s";
-        internalCluster().startDataOnlyNodes(2);
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         Settings settings = Settings.builder()
             .put(indexSettings())
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), invalidRefreshInterval)
@@ -251,7 +285,7 @@ public void testInvalidRefreshInterval() {
     }
 
     public void testCreateIndexWithExplicitNullRefreshInterval() throws ExecutionException, InterruptedException {
-        List<String> dataNodes = internalCluster().startDataOnlyNodes(2);
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         Settings indexSettings = Settings.builder()
             .put(indexSettings())
             .putNull(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey())
@@ -278,7 +312,7 @@ public void testCreateIndexWithExplicitNullRefreshInterval() throws ExecutionExc
      * the index setting. The underlying index should continue to use the same refresh interval as earlier.
      */
     public void testClusterMinimumChangeOnIndexWithCustomRefreshInterval() throws ExecutionException, InterruptedException {
-        List<String> dataNodes = internalCluster().startDataOnlyNodes(2);
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         TimeValue customRefreshInterval = TimeValue.timeValueSeconds(getDefaultRefreshInterval().getSeconds() + randomIntBetween(1, 5));
         Settings indexSettings = Settings.builder()
             .put(indexSettings())