Merge branch 'main' into 12341-bitmap-filtering-push

.github/workflows/benchmark-pull-request.yml

@@ -123,7 +123,7 @@ jobs:
           ref: ${{ env.prHeadRefSha }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Java
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: 21
       - name: Build and Assemble OpenSearch from PR

CHANGELOG-3.0.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
+- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054))
 
 ### Dependencies
 

CHANGELOG.md

@@ -22,6 +22,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998))
 - Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.16.2 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995))
 - Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997))
+- Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110))
+- Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104))
+- Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119))
+- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.8 and `org.bouncycastle:bc-fips` from 1.0.2.5 to 2.0.0 in /distribution/tools/plugin-cli ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103))
 
 ### Changed
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
@@ -33,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908))
 - Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963))
+- Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080))
 
 ### Security
 

README.md

@@ -7,7 +7,7 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.14.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.1")
+[![2.17.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.0")
 [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)

TRIAGING.md

@@ -35,7 +35,7 @@ Meeting structure may vary slightly, but the general structure is as follows:
    - [Core](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+-label%3A%22Search%22%2C%22Search%3ARemote+Search%22%2C%22Search%3AResiliency%22%2C%22Search%3APerformance%22%2C%22Search%3ARelevance%22%2C%22Search%3AAggregations%22%2C%22Search%3AQuery+Capabilities%22%2C%22Search%3AQuery+Insights%22%2C%22Search%3ASearchable+Snapshots%22%2C%22Search%3AUser+Behavior+Insights%22%2C%22Storage%22%2C%22Storage%3AResiliency%22%2C%22Storage%3APerformance%22%2C%22Storage%3ASnapshots%22%2C%22Storage%3ARemote%22%2C%22Storage%3ADurability%22%2C%22Cluster+Manager%22%2C%22ClusterManager%3ARemoteState%22%2C%22Indexing%3AReplication%22%2C%22Indexing%22%2C%22Indexing%3APerformance%22%2C%22Indexing+%26+Search%22)
 5. **Attendee Requests:** An opportunity for any meeting member to request consideration of an issue or pull request.
 6. **Open Discussion:** Attendees can bring up any topics not already covered by filed issues or pull requests.
-7. **Review of Old Untriaged Issues:** Time permitting, each meeting will look at all [untriaged issues older than 14 days](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged+created%3A%3C2024-05-20) to prevent issues from falling through the cracks (note the GitHub API does not allow for relative times, so the date in this search must be updated every meeting).
+7. **Review of Old Untriaged Issues:** Look at all [untriaged issues older than 14 days](https://peternied.github.io/redirect/issue_search.html?owner=opensearch-project&repo=OpenSearch&tag=untriaged&created-since-days=14) to prevent issues from falling through the cracks.
 
 ### What is the role of the facilitator?
 

distribution/tools/plugin-cli/build.gradle

@@ -37,8 +37,8 @@ base {
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
-  api "org.bouncycastle:bcpg-fips:1.0.7.1"
-  api "org.bouncycastle:bc-fips:1.0.2.5"
+  api "org.bouncycastle:bcpg-fips:2.0.8"
+  api "org.bouncycastle:bc-fips:2.0.0"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.3.0'
   testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
@@ -58,33 +58,6 @@ test {
   jvmArgs += [ "-Djava.security.egd=file:/dev/urandom" ]
 }
 
-/*
- * these two classes intentionally use the following JDK internal APIs in order to offer the necessary
- * functionality
- *
- * sun.security.internal.spec.TlsKeyMaterialParameterSpec
- * sun.security.internal.spec.TlsKeyMaterialSpec
- * sun.security.internal.spec.TlsMasterSecretParameterSpec
- * sun.security.internal.spec.TlsPrfParameterSpec
- * sun.security.internal.spec.TlsRsaPremasterSecretParameterSpec
- * sun.security.provider.SecureRandom
- *
- */
-thirdPartyAudit.ignoreViolations(
-  'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator$2'
-)
-
 thirdPartyAudit.ignoreMissingClasses(
   'org.brotli.dec.BrotliInputStream',
   'org.objectweb.asm.AnnotationVisitor',

distribution/tools/plugin-cli/licenses/bc-fips-2.0.0.jar.sha1

@@ -0,0 +1 @@
+ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab

distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.8.jar.sha1

@@ -0,0 +1 @@
+51c2f633e0c32d10de1ebab4c86f93310ff820f8

plugins/ingest-attachment/build.gradle

@@ -66,7 +66,7 @@ dependencies {
   runtimeOnly "com.optimaize.languagedetector:language-detector:0.6"
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   // Other dependencies
-  api 'org.tukaani:xz:1.9'
+  api 'org.tukaani:xz:1.10'
   api "commons-io:commons-io:${versions.commonsio}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 

plugins/ingest-attachment/licenses/xz-1.10.jar.sha1

@@ -0,0 +1 @@
+1be8166f89e035a56c6bfc67dbc423996fe577e2

plugins/repository-hdfs/build.gradle

@@ -66,7 +66,7 @@ dependencies {
   }
   api 'org.apache.htrace:htrace-core4:4.2.0-incubating'
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
-  api 'org.apache.avro:avro:1.11.3'
+  api 'org.apache.avro:avro:1.12.0'
   api 'com.google.code.gson:gson:2.11.0'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
@@ -425,19 +425,6 @@ thirdPartyAudit {
     'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
     'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
     'org.apache.hadoop.shaded.org.xbill.DNS.spi.DNSJavaNameServiceDescriptor',
-
-    'org.apache.avro.reflect.FieldAccessUnsafe',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCachedField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCharField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCustomEncodedField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeDoubleField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeFloatField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeIntField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeLongField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeObjectField',
-    'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField',
   )
 }
 

plugins/repository-hdfs/licenses/avro-1.12.0.jar.sha1

@@ -0,0 +1 @@
+6e692a464b213f6df49f8e3e7fcf42df0dbb7639

rest-api-spec/src/main/resources/rest-api-spec/test/index/110_constant_keyword.yml → rest-api-spec/src/main/resources/rest-api-spec/test/index/115_constant_keyword.yml

@@ -70,8 +70,8 @@
 ---
 "Queries":
   - skip:
-      version: " - 2.99.99"
-      reason: "rangeQuery and regexpQuery are supported in 3.0.0 in main branch"
+      version: " - 2.16.99"
+      reason: "rangeQuery and regexpQuery are introduced in 2.17.0"
 
   - do:
       indices.create:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete_index_template/10_basic.yml

@@ -0,0 +1,52 @@
+setup:
+  - do:
+      indices.put_index_template:
+        name: test_template_1
+        body:
+          index_patterns: test-*
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+          "priority": 50
+
+  - do:
+      indices.put_index_template:
+        name: test_template_2
+        body:
+          index_patterns: test-*
+          data_stream: {}
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+          "priority": 51
+
+---
+teardown:
+  - do:
+      indices.delete_data_stream:
+        name: test-1
+        ignore: 404
+  - do:
+      indices.delete_index_template:
+        name: test_template_1
+        ignore: 404
+  - do:
+      indices.delete_index_template:
+        name: test_template_2
+        ignore: 404
+
+---
+"Delete index template which is not used by data stream but index pattern matches":
+  - skip:
+      version: " - 2.16.99"
+      reason: "fixed in 2.17.0"
+
+  - do:
+      indices.create_data_stream:
+        name: test-1
+
+  - do:
+      indices.delete_index_template:
+        name: test_template_1

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java

@@ -79,7 +79,7 @@ protected boolean forbidPrivateIndexSettings() {
 
     @Before
     public void setup() {
-        asyncUploadMockFsRepo = true;
+        asyncUploadMockFsRepo = false;
     }
 
     public void testCreateCloneIndex() {
@@ -153,6 +153,7 @@ public void testCreateCloneIndex() {
 
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15056")
     public void testCreateCloneIndexLowPriorityRateLimit() {
         Version version = VersionUtils.randomIndexCompatibleVersion(random());
         int numPrimaryShards = 1;
@@ -280,7 +281,7 @@ public void testCreateCloneIndexFailure() throws ExecutionException, Interrupted
             throw new RuntimeException(e);
         } finally {
             setFailRate(REPOSITORY_NAME, 0);
-            ensureGreen();
+            ensureGreen(TimeValue.timeValueSeconds(40));
             // clean up
             client().admin()
                 .cluster()

server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java

@@ -20,6 +20,8 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.index.shard.IndexShard;
@@ -65,11 +67,20 @@ protected Settings featureFlagSettings() {
         return featureSettings.build();
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB);
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString())
+            .build();
+    }
+
     public void testWritableWarmFeatureFlagDisabled() {
         Settings clusterSettings = Settings.builder().put(super.nodeSettings(0)).put(FeatureFlags.TIERED_REMOTE_INDEX, false).build();
         InternalTestCluster internalTestCluster = internalCluster();
         internalTestCluster.startClusterManagerOnlyNode(clusterSettings);
-        internalTestCluster.startDataOnlyNode(clusterSettings);
+        internalTestCluster.startDataAndSearchNodes(1);
 
         Settings indexSettings = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
@@ -94,7 +105,7 @@ public void testWritableWarmFeatureFlagDisabled() {
     public void testWritableWarmBasic() throws Exception {
         InternalTestCluster internalTestCluster = internalCluster();
         internalTestCluster.startClusterManagerOnlyNode();
-        internalTestCluster.startDataOnlyNode();
+        internalTestCluster.startDataAndSearchNodes(1);
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)

server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java

@@ -67,7 +67,6 @@
 import java.util.stream.StreamSupport;
 
 import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
-import static org.opensearch.common.util.FeatureFlags.TIERED_REMOTE_INDEX;
 import static org.opensearch.core.common.util.CollectionUtils.iterableAsArrayList;
 import static org.opensearch.index.store.remote.filecache.FileCacheSettings.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
 import static org.opensearch.test.NodeRoles.clusterManagerOnlyNode;
@@ -1019,11 +1018,12 @@ public void testStartSearchNode() throws Exception {
         internalCluster().startNode(Settings.builder().put(onlyRole(DiscoveryNodeRole.SEARCH_ROLE)));
         // test start node without search role
         internalCluster().startNode(Settings.builder().put(onlyRole(DiscoveryNodeRole.DATA_ROLE)));
-        // test start non-dedicated search node with TIERED_REMOTE_INDEX feature enabled
-        internalCluster().startNode(
-            Settings.builder()
-                .put(onlyRoles(Set.of(DiscoveryNodeRole.SEARCH_ROLE, DiscoveryNodeRole.DATA_ROLE)))
-                .put(TIERED_REMOTE_INDEX, true)
+        // test start non-dedicated search node, if the user doesn't configure the cache size, it fails
+        assertThrows(
+            SettingsException.class,
+            () -> internalCluster().startNode(
+                Settings.builder().put(onlyRoles(Set.of(DiscoveryNodeRole.SEARCH_ROLE, DiscoveryNodeRole.DATA_ROLE)))
+            )
         );
         // test start non-dedicated search node
         assertThrows(

server/src/main/java/org/opensearch/action/search/SearchRequestStats.java

@@ -27,6 +27,7 @@
 @PublicApi(since = "2.11.0")
 public final class SearchRequestStats extends SearchRequestOperationsListener {
     Map<SearchPhaseName, StatsHolder> phaseStatsMap = new EnumMap<>(SearchPhaseName.class);
+    StatsHolder tookStatsHolder;
 
     public static final String SEARCH_REQUEST_STATS_ENABLED_KEY = "search.request_stats_enabled";
     public static final Setting<Boolean> SEARCH_REQUEST_STATS_ENABLED = Setting.boolSetting(
@@ -40,6 +41,7 @@ public final class SearchRequestStats extends SearchRequestOperationsListener {
     public SearchRequestStats(ClusterSettings clusterSettings) {
         this.setEnabled(clusterSettings.get(SEARCH_REQUEST_STATS_ENABLED));
         clusterSettings.addSettingsUpdateConsumer(SEARCH_REQUEST_STATS_ENABLED, this::setEnabled);
+        tookStatsHolder = new StatsHolder();
         for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
             phaseStatsMap.put(searchPhaseName, new StatsHolder());
         }
@@ -57,6 +59,18 @@ public long getPhaseMetric(SearchPhaseName searchPhaseName) {
         return phaseStatsMap.get(searchPhaseName).timing.sum();
     }
 
+    public long getTookCurrent() {
+        return tookStatsHolder.current.count();
+    }
+
+    public long getTookTotal() {
+        return tookStatsHolder.total.count();
+    }
+
+    public long getTookMetric() {
+        return tookStatsHolder.timing.sum();
+    }
+
     @Override
     protected void onPhaseStart(SearchPhaseContext context) {
         phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.inc();
@@ -75,6 +89,23 @@ protected void onPhaseFailure(SearchPhaseContext context, Throwable cause) {
         phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec();
     }
 
+    @Override
+    protected void onRequestStart(SearchRequestContext searchRequestContext) {
+        tookStatsHolder.current.inc();
+    }
+
+    @Override
+    protected void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) {
+        tookStatsHolder.current.dec();
+        tookStatsHolder.total.inc();
+        tookStatsHolder.timing.inc(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos()));
+    }
+
+    @Override
+    protected void onRequestFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) {
+        tookStatsHolder.current.dec();
+    }
+
     /**
      * Holder of statistics values
      *

server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -944,7 +944,7 @@ static ClusterState innerRemoveIndexTemplateV2(ClusterState currentState, String
 
     static Set<String> dataStreamsUsingTemplate(final ClusterState state, final String templateName) {
         final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName);
-        if (template == null) {
+        if (template == null || template.getDataStreamTemplate() == null) {
             return Collections.emptySet();
         }
         final Set<String> dataStreams = state.metadata().dataStreams().keySet();