From 00947fce98b33e0b2b50cd887c9ac4e4d0ffba90 Mon Sep 17 00:00:00 2001
From: Peter Nied <petern@amazon.com>
Date: Wed, 1 May 2024 22:04:19 -0500
Subject: [PATCH 01/52] Revert "Skip running gradle checks on release notes
 (#13477)" (#13499)

This reverts commit 1765e20cf3939cb0fff0b8af37294da26d9cbea0.
---
 .github/workflows/gradle-check.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
index 0921aff721836..1f5c187c28e7d 100644
--- a/.github/workflows/gradle-check.yml
+++ b/.github/workflows/gradle-check.yml
@@ -7,10 +7,6 @@ on:
       - 'dependabot/**'
   pull_request_target:
     types: [opened, synchronize, reopened]
-    paths-ignore:
-      - 'release-notes/**'
-      - '.github/**'
-      - '**.md'
 
 permissions:
   contents: read # to fetch code (actions/checkout)

From ef841dd16878fb60b082dccd8faa8d9099e61296 Mon Sep 17 00:00:00 2001
From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com>
Date: Thu, 2 May 2024 12:51:45 +0530
Subject: [PATCH 02/52] [Remote Migration] Handle shard allocation for none
 migration direction (#13322)

Signed-off-by: Lakshya Taragi <lakshya.taragi@gmail.com>
---
 .../RemoteMigrationAllocationDeciderIT.java   | 409 +++++++++++++++++-
 .../RemotePrimaryRelocationIT.java            |   8 +
 ...eMigrationShardAllocationBaseTestCase.java | 165 ++++++-
 .../metadata/MetadataCreateIndexService.java  |   2 +-
 ...RemoteStoreMigrationAllocationDecider.java |  48 +-
 ...eStoreMigrationAllocationDeciderTests.java | 337 +++++++--------
 6 files changed, 753 insertions(+), 216 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java
index de425ffc63816..eeb6a5a5626e4 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java
@@ -11,8 +11,11 @@
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.cluster.routing.allocation.decider.Decision;
 import org.opensearch.common.Priority;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -21,13 +24,17 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
 
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class RemoteMigrationAllocationDeciderIT extends MigrationBaseTestCase {
+public class RemoteMigrationAllocationDeciderIT extends RemoteStoreMigrationShardAllocationBaseTestCase {
 
     // When the primary is on doc rep node, existing replica copy can get allocated on excluded docrep node.
     public void testFilterAllocationSkipsReplica() throws IOException {
@@ -127,4 +134,404 @@ public void testFilterAllocationSkipsReplicaOnExcludedNode() throws IOException
         assertTrue(clusterHealthResponse.isTimedOut());
         ensureYellow("test");
     }
+
+    // When under mixed mode and remote_store direction, a primary shard can only be allocated to a remote node
+
+    public void testNewPrimaryShardAllocationForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add non-remote data node");
+        String nonRemoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode = assertNodeInCluster(nonRemoteNodeName);
+
+        logger.info("Set mixed mode and remote_store direction");
+        setClusterMode(MIXED.mode);
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Verify expected decision for allocating a new primary shard on a non-remote node");
+        prepareIndexWithoutReplica(Optional.empty());
+        Decision decision = getDecisionForTargetNode(nonRemoteNode, true, true, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: primary shard copy can not be allocated to a non-remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt allocation on non-remote node");
+        attemptAllocation(null);
+
+        logger.info("Verify non-allocation of primary shard on non-remote node");
+        assertNonAllocation(true);
+
+        logger.info("Add remote data node");
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode = assertNodeInCluster(remoteNodeName);
+
+        logger.info("Verify expected decision for allocating a new primary shard on a remote node");
+        excludeAllNodes();
+        decision = getDecisionForTargetNode(remoteNode, true, true, false);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: primary shard copy can be allocated to a remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt free allocation");
+        attemptAllocation(null);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Verify allocation of primary shard on remote node");
+        assertAllocation(true, remoteNode);
+    }
+
+    // When under mixed mode and remote_store direction, a replica shard can only be allocated to a remote node if the primary has relocated
+    // to another remote node
+
+    public void testNewReplicaShardAllocationIfPrimaryShardOnNonRemoteNodeForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add non-remote data node");
+        String nonRemoteNodeName1 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode1 = assertNodeInCluster(nonRemoteNodeName1);
+
+        logger.info("Allocate primary shard on non-remote node");
+        prepareIndexWithAllocatedPrimary(nonRemoteNode1, Optional.empty());
+
+        logger.info("Add remote data node");
+        setClusterMode(MIXED.mode);
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode = assertNodeInCluster(remoteNodeName);
+
+        logger.info("Set remote_store direction");
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Verify expected decision for allocating a replica shard on a remote node");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(remoteNode, false, true, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can not be allocated to a remote node since primary shard copy is not yet migrated to remote",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt free allocation of replica shard");
+        attemptAllocation(null);
+
+        logger.info("Verify non-allocation of replica shard");
+        assertNonAllocation(false);
+
+        logger.info("Add another non-remote data node");
+        setAddRemote(false);
+        String nonRemoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode2 = assertNodeInCluster(nonRemoteNodeName2);
+
+        logger.info("Verify expected decision for allocating the replica shard on a non-remote node");
+        excludeAllNodes();
+        decision = getDecisionForTargetNode(nonRemoteNode2, false, true, false);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can be allocated to a non-remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt free allocation of replica shard");
+        attemptAllocation(null);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Verify allocation of replica shard on non-remote node");
+        assertAllocation(false, nonRemoteNode2);
+    }
+
+    public void testNewReplicaShardAllocationIfPrimaryShardOnRemoteNodeForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add non-remote data nodes");
+        String nonRemoteNodeName1 = internalCluster().startDataOnlyNode();
+        String nonRemoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode1 = assertNodeInCluster(nonRemoteNodeName1);
+        DiscoveryNode nonRemoteNode2 = assertNodeInCluster(nonRemoteNodeName2);
+
+        logger.info("Allocate primary and replica shard on non-remote nodes");
+        createIndex(TEST_INDEX, 1);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Set mixed mode");
+        setClusterMode(MIXED.mode);
+
+        logger.info("Add remote data nodes");
+        setAddRemote(true);
+        String remoteNodeName1 = internalCluster().startDataOnlyNode();
+        String remoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode1 = assertNodeInCluster(remoteNodeName1);
+        DiscoveryNode remoteNode2 = assertNodeInCluster(remoteNodeName2);
+
+        logger.info("Set remote_store direction");
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Relocate primary shard to remote node");
+        DiscoveryNode initialPrimaryNode = primaryNodeName(TEST_INDEX).equals(nonRemoteNodeName1) ? nonRemoteNode1 : nonRemoteNode2;
+        DiscoveryNode initialReplicaNode = initialPrimaryNode.equals(nonRemoteNode1) ? nonRemoteNode2 : nonRemoteNode1;
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(TEST_INDEX, 0, initialPrimaryNode.getName(), remoteNodeName1))
+                .get()
+        );
+        ensureGreen(TEST_INDEX);
+        assertAllocation(true, remoteNode1);
+
+        logger.info("Verify expected decision for relocating a replica shard on non-remote node");
+        Decision decision = getDecisionForTargetNode(initialPrimaryNode, false, true, true);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can be relocated to a non-remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt relocation of replica shard to non-remote node");
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(TEST_INDEX, 0, initialReplicaNode.getName(), initialPrimaryNode.getName()))
+                .get()
+        );
+
+        logger.info("Verify relocation of replica shard to non-remote node");
+        ensureGreen(TEST_INDEX);
+        assertAllocation(false, initialPrimaryNode);
+
+        logger.info("Verify expected decision for relocating a replica shard on remote node");
+        decision = getDecisionForTargetNode(remoteNode2, false, true, true);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can be relocated to a remote node since primary shard copy has been migrated to remote",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt relocation of replica shard to remote node");
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(TEST_INDEX, 0, initialPrimaryNode.getName(), remoteNodeName2))
+                .get()
+        );
+
+        logger.info("Verify relocation of replica shard to non-remote node");
+        ensureGreen(TEST_INDEX);
+        assertAllocation(false, remoteNode2);
+    }
+
+    // When under strict mode, a shard can be allocated to any node
+
+    public void testAlwaysAllocateNewShardForStrictMode() throws Exception {
+        boolean isRemoteCluster = randomBoolean();
+        boolean isReplicaAllocation = randomBoolean();
+
+        logger.info("Initialize cluster and add nodes");
+        setAddRemote(isRemoteCluster);
+        internalCluster().startClusterManagerOnlyNode();
+        String nodeName1 = internalCluster().startDataOnlyNode();
+        String nodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode node1 = assertNodeInCluster(nodeName1);
+        DiscoveryNode node2 = assertNodeInCluster(nodeName2);
+
+        if (isReplicaAllocation) {
+            prepareIndexWithAllocatedPrimary(node1, Optional.empty());
+        } else {
+            prepareIndexWithoutReplica(Optional.empty());
+        }
+
+        if (isRemoteCluster) {
+            assertRemoteStoreBackedIndex(TEST_INDEX);
+        } else {
+            assertNonRemoteStoreBackedIndex(TEST_INDEX);
+        }
+
+        logger.info("Verify expected decision for allocation of a shard");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(
+            isReplicaAllocation ? node2 : randomFrom(node1, node2),
+            !isReplicaAllocation,
+            true,
+            false
+        );
+        assertEquals(Decision.Type.YES, decision.type());
+        String expectedReason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a %s node for strict compatibility mode",
+            (isReplicaAllocation ? "replica" : "primary"),
+            (isRemoteCluster ? "remote" : "non-remote")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt free allocation");
+        attemptAllocation(null);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Verify allocation of shard");
+        assertAllocation(!isReplicaAllocation, !isReplicaAllocation ? null : node2);
+    }
+
+    // When under mixed mode and remote_store direction, shard of a remote store backed index can not be allocated to a non-remote node
+
+    public void testRemoteStoreBackedIndexShardAllocationForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Set mixed mode");
+        setClusterMode(MIXED.mode);
+
+        logger.info("Add remote and non-remote nodes");
+        String nonRemoteNodeName = internalCluster().startDataOnlyNode();
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode = assertNodeInCluster(remoteNodeName);
+        DiscoveryNode nonRemoteNode = assertNodeInCluster(nonRemoteNodeName);
+
+        logger.info("Set remote_store direction");
+        setDirection(REMOTE_STORE.direction);
+
+        boolean isReplicaAllocation = randomBoolean();
+        if (isReplicaAllocation) {
+            logger.info("Create index with primary allocated on remote node");
+            prepareIndexWithAllocatedPrimary(remoteNode, Optional.empty());
+        } else {
+            logger.info("Create index with unallocated primary");
+            prepareIndexWithoutReplica(Optional.empty());
+        }
+
+        logger.info("Verify remote store backed index");
+        assertRemoteStoreBackedIndex(TEST_INDEX);
+
+        logger.info("Verify expected decision for allocation of shard on a non-remote node");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(nonRemoteNode, !isReplicaAllocation, false, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        String expectedReason = String.format(
+            Locale.ROOT,
+            "[remote_store migration_direction]: %s shard copy can not be allocated to a non-remote node because a remote store backed index's shard copy can only be allocated to a remote node",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt allocation of shard on non-remote node");
+        attemptAllocation(nonRemoteNodeName);
+
+        logger.info("Verify non-allocation of shard");
+        assertNonAllocation(!isReplicaAllocation);
+    }
+
+    // When under mixed mode and none direction, allocate shard of a remote store backed index to a remote node and shard of a non remote
+    // store backed index to a non-remote node only
+
+    public void testAllocationForNoneDirectionAndMixedMode() throws Exception {
+        boolean isRemoteStoreBackedIndex = randomBoolean();
+        boolean isReplicaAllocation = randomBoolean();
+        logger.info(
+            String.format(
+                Locale.ROOT,
+                "Test for allocation decisions for %s shard of a %s store backed index under NONE direction",
+                (isReplicaAllocation ? "replica" : "primary"),
+                (isRemoteStoreBackedIndex ? "remote" : "non remote")
+            )
+        );
+
+        logger.info("Initialize cluster");
+        setAddRemote(isRemoteStoreBackedIndex);
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add data nodes");
+        String previousNodeName1 = internalCluster().startDataOnlyNode();
+        String previousNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode previousNode1 = assertNodeInCluster(previousNodeName1);
+        DiscoveryNode previousNode2 = assertNodeInCluster(previousNodeName2);
+
+        logger.info("Prepare test index");
+        if (isReplicaAllocation) {
+            prepareIndexWithAllocatedPrimary(previousNode1, Optional.empty());
+        } else {
+            prepareIndexWithoutReplica(Optional.empty());
+        }
+
+        if (isRemoteStoreBackedIndex) {
+            assertRemoteStoreBackedIndex(TEST_INDEX);
+        } else {
+            assertNonRemoteStoreBackedIndex(TEST_INDEX);
+        }
+
+        logger.info("Switch to MIXED cluster compatibility mode");
+        setClusterMode(MIXED.mode);
+        setAddRemote(!addRemote);
+        String newNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode newNode = assertNodeInCluster(newNodeName);
+
+        logger.info("Verify decision for allocation on the new node");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(newNode, !isReplicaAllocation, false, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        String expectedReason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can not be allocated to a %s node for %s store backed index",
+            (isReplicaAllocation ? "replica" : "primary"),
+            (isRemoteStoreBackedIndex ? "non-remote" : "remote"),
+            (isRemoteStoreBackedIndex ? "remote" : "non remote")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt allocation of shard on new node");
+        attemptAllocation(newNodeName);
+
+        logger.info("Verify non-allocation of shard");
+        assertNonAllocation(!isReplicaAllocation);
+
+        logger.info("Verify decision for allocation on previous node");
+        decision = getDecisionForTargetNode(previousNode2, !isReplicaAllocation, true, false);
+        assertEquals(Decision.Type.YES, decision.type());
+        expectedReason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a %s node for %s store backed index",
+            (isReplicaAllocation ? "replica" : "primary"),
+            (isRemoteStoreBackedIndex ? "remote" : "non-remote"),
+            (isRemoteStoreBackedIndex ? "remote" : "non remote")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt free allocation of shard");
+        attemptAllocation(null);
+
+        logger.info("Verify successful allocation of shard");
+        if (!isReplicaAllocation) {
+            ensureGreen(TEST_INDEX);
+        } else {
+            ensureYellowAndNoInitializingShards(TEST_INDEX);
+        }
+        assertAllocation(!isReplicaAllocation, null);
+        logger.info("Verify allocation on one of the previous nodes");
+        ShardRouting shardRouting = getShardRouting(!isReplicaAllocation);
+        assertTrue(
+            shardRouting.currentNodeId().equals(previousNode1.getId()) || shardRouting.currentNodeId().equals(previousNode2.getId())
+        );
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java
index 8f6c1e2d9a68c..293691ace2edd 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java
@@ -91,6 +91,10 @@ public void testRemotePrimaryRelocation() throws Exception {
         int finalCurrentDoc1 = currentDoc;
         waitUntil(() -> numAutoGenDocs.get() > finalCurrentDoc1 + 5);
 
+        // Change direction to remote store
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
         logger.info("-->  relocating from {} to {} ", docRepNodes, remoteNode);
         client().admin()
             .cluster()
@@ -179,6 +183,10 @@ public void testMixedModeRelocation_RemoteSeedingFail() throws Exception {
             .setTransientSettings(Settings.builder().put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), "10s"))
             .get();
 
+        // Change direction to remote store
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
         logger.info("--> relocating from {} to {} ", docRepNode, remoteNode);
         client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, docRepNode, remoteNode)).execute().actionGet();
         ClusterHealthResponse clusterHealthResponse = client().admin()
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
index ffcab9483485d..cf689aa554c8b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
@@ -8,13 +8,21 @@
 
 package org.opensearch.remotemigration;
 
+import org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.opensearch.cluster.routing.allocation.MoveDecision;
+import org.opensearch.cluster.routing.allocation.NodeAllocationResult;
+import org.opensearch.cluster.routing.allocation.decider.Decision;
+import org.opensearch.common.Nullable;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.index.IndexSettings;
@@ -22,6 +30,7 @@
 import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.snapshots.SnapshotState;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -47,7 +56,7 @@ protected void setClusterMode(String mode) {
     }
 
     // set the migration direction for cluster [remote_store, docrep, none]
-    public void setDirection(String direction) {
+    protected void setDirection(String direction) {
         updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), direction));
         assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
     }
@@ -79,7 +88,7 @@ protected String allNodesExcept(String except) {
         return exclude.toString();
     }
 
-    // create a new test index
+    // create a new test index with un-allocated primary and no replicas
     protected void prepareIndexWithoutReplica(Optional<String> name) {
         String indexName = name.orElse(TEST_INDEX);
         internalCluster().client()
@@ -96,6 +105,33 @@ protected void prepareIndexWithoutReplica(Optional<String> name) {
             .actionGet();
     }
 
+    // create a new test index with allocated primary and 1 unallocated replica
+    public void prepareIndexWithAllocatedPrimary(DiscoveryNode primaryShardNode, Optional<String> name) {
+        String indexName = name.orElse(TEST_INDEX);
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareCreate(indexName)
+            .setSettings(
+                Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 1)
+                    .put("index.routing.allocation.include._name", primaryShardNode.getName())
+                    .put("index.routing.allocation.exclude._name", allNodesExcept(primaryShardNode.getName()))
+            )
+            .setWaitForActiveShards(ActiveShardCount.ONE)
+            .execute()
+            .actionGet();
+
+        ensureYellowAndNoInitializingShards(TEST_INDEX);
+
+        logger.info(" --> verify allocation of primary shard");
+        assertAllocation(true, primaryShardNode);
+
+        logger.info(" --> verify non-allocation of replica shard");
+        assertNonAllocation(false);
+    }
+
     protected ShardRouting getShardRouting(boolean isPrimary) {
         IndexShardRoutingTable table = internalCluster().client()
             .admin()
@@ -110,6 +146,130 @@ protected ShardRouting getShardRouting(boolean isPrimary) {
         return (isPrimary ? table.primaryShard() : table.replicaShards().get(0));
     }
 
+    // obtain decision for allocation/relocation of a shard to a given node
+    protected Decision getDecisionForTargetNode(
+        DiscoveryNode targetNode,
+        boolean isPrimary,
+        boolean includeYesDecisions,
+        boolean isRelocation
+    ) {
+        ClusterAllocationExplanation explanation = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareAllocationExplain()
+            .setIndex(TEST_INDEX)
+            .setShard(0)
+            .setPrimary(isPrimary)
+            .setIncludeYesDecisions(includeYesDecisions)
+            .get()
+            .getExplanation();
+
+        Decision requiredDecision = null;
+        List<NodeAllocationResult> nodeAllocationResults;
+        if (isRelocation) {
+            MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+            nodeAllocationResults = moveDecision.getNodeDecisions();
+        } else {
+            AllocateUnassignedDecision allocateUnassignedDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+            nodeAllocationResults = allocateUnassignedDecision.getNodeDecisions();
+        }
+
+        for (NodeAllocationResult nodeAllocationResult : nodeAllocationResults) {
+            if (nodeAllocationResult.getNode().equals(targetNode)) {
+                for (Decision decision : nodeAllocationResult.getCanAllocateDecision().getDecisions()) {
+                    if (decision.label().equals(NAME)) {
+                        requiredDecision = decision;
+                        break;
+                    }
+                }
+            }
+        }
+
+        assertNotNull(requiredDecision);
+        return requiredDecision;
+    }
+
+    // get allocation and relocation decisions for all nodes
+    protected void excludeAllNodes() {
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings(TEST_INDEX)
+                .setSettings(
+                    Settings.builder()
+                        .put("index.routing.allocation.include._name", "")
+                        .put("index.routing.allocation.exclude._name", allNodesExcept(null))
+                )
+                .execute()
+                .actionGet()
+        );
+    }
+
+    protected void includeAllNodes() {
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings(TEST_INDEX)
+                .setSettings(
+                    Settings.builder()
+                        .put("index.routing.allocation.exclude._name", "")
+                        .put("index.routing.allocation.include._name", allNodesExcept(null))
+                )
+                .execute()
+                .actionGet()
+        );
+    }
+
+    protected void attemptAllocation(@Nullable String targetNodeName) {
+        Settings.Builder settingsBuilder;
+        if (targetNodeName != null) {
+            settingsBuilder = Settings.builder()
+                .put("index.routing.allocation.include._name", targetNodeName)
+                .put("index.routing.allocation.exclude._name", allNodesExcept(targetNodeName));
+        } else {
+            String clusterManagerNodeName = internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareState()
+                .execute()
+                .actionGet()
+                .getState()
+                .getNodes()
+                .getClusterManagerNode()
+                .getName();
+            // to allocate freely among all nodes other than cluster-manager node
+            settingsBuilder = Settings.builder()
+                .put("index.routing.allocation.include._name", allNodesExcept(clusterManagerNodeName))
+                .put("index.routing.allocation.exclude._name", clusterManagerNodeName);
+        }
+        internalCluster().client().admin().indices().prepareUpdateSettings(TEST_INDEX).setSettings(settingsBuilder).execute().actionGet();
+    }
+
+    // verify that shard does not exist at targetNode
+    protected void assertNonAllocation(boolean isPrimary) {
+        if (isPrimary) {
+            ensureRed(TEST_INDEX);
+        } else {
+            ensureYellowAndNoInitializingShards(TEST_INDEX);
+        }
+        ShardRouting shardRouting = getShardRouting(isPrimary);
+        assertFalse(shardRouting.active());
+        assertNull(shardRouting.currentNodeId());
+        assertEquals(ShardRoutingState.UNASSIGNED, shardRouting.state());
+    }
+
+    // verify that shard exists at targetNode
+    protected void assertAllocation(boolean isPrimary, @Nullable DiscoveryNode targetNode) {
+        ShardRouting shardRouting = getShardRouting(isPrimary);
+        assertTrue(shardRouting.active());
+        assertNotNull(shardRouting.currentNodeId());
+        if (targetNode != null) {
+            assertEquals(targetNode.getId(), shardRouting.currentNodeId());
+        }
+    }
+
     // create a snapshot
     public static SnapshotInfo createSnapshot(String snapshotRepoName, String snapshotName, String... indices) {
         SnapshotInfo snapshotInfo = internalCluster().client()
@@ -194,4 +354,5 @@ public static void assertRemoteStoreBackedIndex(String indexName) {
             INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
         );
     }
+
 }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index b31985a260361..121f8d935cf48 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1664,7 +1664,7 @@ public static void validateRefreshIntervalSettings(Settings requestSettings, Clu
      * @param clusterSettings cluster setting
      */
     static void validateTranslogDurabilitySettings(Settings requestSettings, ClusterSettings clusterSettings, Settings settings) {
-        if (isRemoteDataAttributePresent(settings) == false
+        if ((isRemoteDataAttributePresent(settings) == false && isMigratingToRemoteStore(clusterSettings) == false)
             || IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.exists(requestSettings) == false
             || clusterSettings.get(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING) == false) {
             return;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
index 7d40aacb71e25..4fc5fff805663 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
@@ -95,32 +95,38 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             );
         }
 
-        if (migrationDirection.equals(Direction.REMOTE_STORE) == false) {
-            // docrep migration direction is currently not supported
+        IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
+        boolean remoteSettingsBackedIndex = IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexMetadata.getSettings());
+
+        if (migrationDirection.equals(Direction.NONE)) {
+            // remote backed indices on docrep nodes and non remote backed indices on remote nodes are not allowed
+            boolean isNoDecision = remoteSettingsBackedIndex ^ targetNode.isRemoteStoreNode();
+            String reason = String.format(Locale.ROOT, " for %sremote store backed index", remoteSettingsBackedIndex ? "" : "non ");
             return allocation.decision(
-                Decision.YES,
+                isNoDecision ? Decision.NO : Decision.YES,
                 NAME,
-                getDecisionDetails(true, shardRouting, targetNode, " for non remote_store direction")
+                getDecisionDetails(!isNoDecision, shardRouting, targetNode, reason)
             );
-        }
-
-        // check for remote store backed indices
-        IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
-        boolean remoteStoreBackedIndex = IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexMetadata.getSettings());
-        if (remoteStoreBackedIndex && targetNode.isRemoteStoreNode() == false) {
-            // allocations and relocations must be to a remote node
-            String reason = String.format(
-                Locale.ROOT,
-                " because a remote store backed index's shard copy can only be %s to a remote node",
-                ((shardRouting.assignedToNode() == false) ? "allocated" : "relocated")
-            );
-            return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason));
-        }
+        } else if (migrationDirection.equals(Direction.DOCREP)) {
+            // docrep migration direction is currently not supported
+            return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, shardRouting, targetNode, " for DOCREP direction"));
+        } else {
+            // check for remote store backed indices
+            if (remoteSettingsBackedIndex && targetNode.isRemoteStoreNode() == false) {
+                // allocations and relocations must be to a remote node
+                String reason = String.format(
+                    Locale.ROOT,
+                    " because a remote store backed index's shard copy can only be %s to a remote node",
+                    ((shardRouting.assignedToNode() == false) ? "allocated" : "relocated")
+                );
+                return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason));
+            }
 
-        if (shardRouting.primary()) {
-            return primaryShardDecision(shardRouting, targetNode, allocation);
+            if (shardRouting.primary()) {
+                return primaryShardDecision(shardRouting, targetNode, allocation);
+            }
+            return replicaShardDecision(shardRouting, targetNode, allocation);
         }
-        return replicaShardDecision(shardRouting, targetNode, allocation);
     }
 
     // handle scenarios for allocation of a new shard's primary copy
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java
index 3e130a42952e4..ee4dbe9738e04 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java
@@ -70,6 +70,8 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.NONE;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.hamcrest.core.Is.is;
@@ -89,7 +91,7 @@ public class RemoteStoreMigrationAllocationDeciderTests extends OpenSearchAlloca
         .build();
 
     private final Settings remoteStoreDirectionSettings = Settings.builder()
-        .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE)
+        .put(MIGRATION_DIRECTION_SETTING.getKey(), REMOTE_STORE)
         .build();
     private final Settings docrepDirectionSettings = Settings.builder()
         .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.DOCREP)
@@ -106,7 +108,9 @@ public class RemoteStoreMigrationAllocationDeciderTests extends OpenSearchAlloca
     private Metadata metadata;
     private RoutingTable routingTable = null;
 
-    private void beforeAllocation() {
+    private ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
+
+    private void beforeAllocation(String direction) {
         FeatureFlags.initializeFeatureFlags(directionEnabledNodeSettings);
         if (isRemoteStoreBackedIndex == null) {
             isRemoteStoreBackedIndex = randomBoolean();
@@ -116,11 +120,7 @@ private void beforeAllocation() {
         String compatibilityMode = isMixedMode
             ? RemoteStoreNodeService.CompatibilityMode.MIXED.mode
             : RemoteStoreNodeService.CompatibilityMode.STRICT.mode;
-        customSettings = getCustomSettings(
-            RemoteStoreNodeService.Direction.REMOTE_STORE.direction,
-            compatibilityMode,
-            indexMetadataBuilder
-        );
+        customSettings = getCustomSettings(direction, compatibilityMode, indexMetadataBuilder);
 
         if (routingTable != null) {
             metadata = Metadata.builder().put(indexMetadataBuilder).build();
@@ -149,6 +149,35 @@ private void beforeAllocation() {
         routingAllocation.debugDecision(true);
     }
 
+    private void prepareRoutingTable(boolean isReplicaAllocation, String primaryShardNodeId) {
+        routingTable = RoutingTable.builder()
+            .add(
+                IndexRoutingTable.builder(shardId.getIndex())
+                    .addIndexShard(
+                        new IndexShardRoutingTable.Builder(shardId).addShard(
+                            TestShardRouting.newShardRouting(
+                                shardId.getIndexName(),
+                                shardId.getId(),
+                                (isReplicaAllocation ? primaryShardNodeId : null),
+                                true,
+                                (isReplicaAllocation ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED)
+                            )
+                        )
+                            .addShard(
+                                TestShardRouting.newShardRouting(
+                                    shardId.getIndexName(),
+                                    shardId.getId(),
+                                    null,
+                                    false,
+                                    ShardRoutingState.UNASSIGNED
+                                )
+                            )
+                            .build()
+                    )
+            )
+            .build();
+    }
+
     // tests for primary shard copy allocation with MIXED mode and REMOTE_STORE direction
 
     public void testDontAllocateNewPrimaryShardOnNonRemoteNodeForMixedModeAndRemoteStoreDirection() {
@@ -166,7 +195,7 @@ public void testDontAllocateNewPrimaryShardOnNonRemoteNodeForMixedModeAndRemoteS
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         ShardRouting primaryShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
         RoutingNode nonRemoteRoutingNode = clusterState.getRoutingNodes().node(nonRemoteNode.getId());
@@ -196,7 +225,7 @@ public void testAllocateNewPrimaryShardOnRemoteNodeForMixedModeAndRemoteStoreDir
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         ShardRouting primaryShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
         RoutingNode remoteRoutingNode = clusterState.getRoutingNodes().node(remoteNode.getId());
@@ -216,39 +245,11 @@ public void testDontAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnNonRemote
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode nonRemoteNode = getNonRemoteNode();
         DiscoveryNode remoteNode = getRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary on non-remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                nonRemoteNode.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary on non-remote node, new replica's allocation
+        prepareRoutingTable(true, nonRemoteNode.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(nonRemoteNode)
@@ -257,7 +258,7 @@ public void testDontAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnNonRemote
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
         ShardRouting replicaShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
@@ -278,40 +279,12 @@ public void testAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnRemoteNodeFor
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode remoteNode1 = getRemoteNode();
         DiscoveryNode remoteNode2 = getRemoteNode();
         DiscoveryNode nonRemoteNode = getNonRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary on remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                remoteNode1.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary on remote node, new replica's allocation
+        prepareRoutingTable(true, remoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(remoteNode1)
@@ -322,7 +295,7 @@ public void testAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnRemoteNodeFor
             .localNodeId(nonRemoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
         ShardRouting replicaShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
@@ -343,40 +316,12 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnNonRemoteN
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode remoteNode = getRemoteNode();
         DiscoveryNode nonRemoteNode1 = getNonRemoteNode();
         DiscoveryNode nonRemoteNode2 = getNonRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary shard on non-remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                nonRemoteNode1.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary shard on non-remote node, new replica's allocation
+        prepareRoutingTable(true, nonRemoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(remoteNode)
@@ -387,7 +332,7 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnNonRemoteN
             .localNodeId(nonRemoteNode2.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
 
@@ -411,39 +356,11 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnRemoteNode
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode nonRemoteNode = getNonRemoteNode();
         DiscoveryNode remoteNode = getRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary shard on non-remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                remoteNode.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary shard on remote node, new replica's allocation
+        prepareRoutingTable(true, remoteNode.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(nonRemoteNode)
@@ -452,7 +369,7 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnRemoteNode
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
         ShardRouting replicaShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
@@ -478,39 +395,12 @@ public void testAlwaysAllocateNewShardForStrictMode() {
         isMixedMode = false;
         isRemoteStoreBackedIndex = false;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode nonRemoteNode1 = getNonRemoteNode();
         DiscoveryNode nonRemoteNode2 = getNonRemoteNode();
 
         boolean isReplicaAllocation = randomBoolean();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                (isReplicaAllocation ? nonRemoteNode1.getId() : null),
-                                true,
-                                (isReplicaAllocation ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED)
-                            )
-                        )
-                            .addShard(
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        prepareRoutingTable(isReplicaAllocation, nonRemoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(nonRemoteNode1)
@@ -519,7 +409,7 @@ public void testAlwaysAllocateNewShardForStrictMode() {
             .localNodeId(nonRemoteNode2.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
 
@@ -543,33 +433,7 @@ public void testAlwaysAllocateNewShardForStrictMode() {
         DiscoveryNode remoteNode1 = getRemoteNode();
         DiscoveryNode remoteNode2 = getRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                (isReplicaAllocation ? remoteNode1.getId() : null),
-                                true,
-                                (isReplicaAllocation ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED)
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        prepareRoutingTable(isReplicaAllocation, remoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(remoteNode1)
@@ -578,7 +442,7 @@ public void testAlwaysAllocateNewShardForStrictMode() {
             .localNodeId(remoteNode2.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
 
@@ -598,6 +462,97 @@ public void testAlwaysAllocateNewShardForStrictMode() {
         assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
     }
 
+    // test for NONE direction
+    public void testAllocationForNoneDirection() {
+        shardCount = 1;
+        replicaCount = 1;
+        isMixedMode = true;
+        isRemoteStoreBackedIndex = false; // non-remote store backed index
+
+        DiscoveryNode remoteNode1 = getRemoteNode();
+        DiscoveryNode remoteNode2 = getRemoteNode();
+        DiscoveryNode nonRemoteNode1 = getNonRemoteNode();
+        DiscoveryNode nonRemoteNode2 = getNonRemoteNode();
+
+        boolean isReplicaAllocation = randomBoolean();
+
+        prepareRoutingTable(isReplicaAllocation, nonRemoteNode1.getId());
+
+        discoveryNodes = DiscoveryNodes.builder()
+            .add(remoteNode1)
+            .localNodeId(remoteNode1.getId())
+            .add(remoteNode2)
+            .localNodeId(remoteNode2.getId())
+            .add(nonRemoteNode1)
+            .localNodeId(nonRemoteNode1.getId())
+            .add(nonRemoteNode2)
+            .localNodeId(nonRemoteNode2.getId())
+            .build();
+
+        beforeAllocation(NONE.direction);
+        assertEquals(2, clusterState.getRoutingTable().allShards().size());
+
+        ShardRouting shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
+        if (isReplicaAllocation) {
+            shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
+        }
+        RoutingNode nonRemoteRoutingNode = clusterState.getRoutingNodes().node(nonRemoteNode2.getId());
+        RoutingNode remoteRoutingNode = clusterState.getRoutingNodes().node(remoteNode2.getId());
+
+        // allocation decision for non-remote node for non-remote store backed index
+        Decision decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, nonRemoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.YES));
+        String reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a non-remote node for non remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+
+        // allocation decision for remote node for non-remote store backed index
+        decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, remoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.NO));
+        reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can not be allocated to a remote node for non remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+
+        isRemoteStoreBackedIndex = true; // remote store backed index
+        prepareRoutingTable(isReplicaAllocation, remoteNode1.getId());
+
+        beforeAllocation(NONE.direction);
+        assertEquals(2, clusterState.getRoutingTable().allShards().size());
+
+        shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
+        if (isReplicaAllocation) {
+            shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
+        }
+        nonRemoteRoutingNode = clusterState.getRoutingNodes().node(nonRemoteNode2.getId());
+        remoteRoutingNode = clusterState.getRoutingNodes().node(remoteNode2.getId());
+
+        // allocation decision for non-remote node for remote store backed index
+        decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, nonRemoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.NO));
+        reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can not be allocated to a non-remote node for remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+
+        // allocation decision for remote node for remote store backed index
+        decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, remoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.YES));
+        reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a remote node for remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+    }
+
     // prepare index metadata for test-index
     private IndexMetadata.Builder getIndexMetadataBuilder(boolean isRemoteStoreBackedIndex, int shardCount, int replicaCount) {
         Settings.Builder builder = settings(Version.CURRENT);
@@ -614,7 +569,7 @@ private IndexMetadata.Builder getIndexMetadataBuilder(boolean isRemoteStoreBacke
     private Settings getCustomSettings(String direction, String compatibilityMode, IndexMetadata.Builder indexMetadataBuilder) {
         Settings.Builder builder = Settings.builder();
         // direction settings
-        if (direction.toLowerCase(Locale.ROOT).equals(RemoteStoreNodeService.Direction.REMOTE_STORE.direction)) {
+        if (direction.toLowerCase(Locale.ROOT).equals(REMOTE_STORE.direction)) {
             builder.put(remoteStoreDirectionSettings);
         } else if (direction.toLowerCase(Locale.ROOT).equals(RemoteStoreNodeService.Direction.DOCREP.direction)) {
             builder.put(docrepDirectionSettings);

From 605331012b713baf94bb184c1f3a545198ad91b6 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 2 May 2024 11:07:40 -0400
Subject: [PATCH 03/52] Bump 2.x to 2.15.0 (#13495) (#13506)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 .ci/bwcVersions                                     | 1 +
 libs/core/src/main/java/org/opensearch/Version.java | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index f3a9aa2787a80..e490c6ae6b4ea 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -32,3 +32,4 @@ BWC_VERSION:
   - "2.13.0"
   - "2.13.1"
   - "2.14.0"
+  - "2.15.0"
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index f312c484a4842..823eb110eaa10 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -103,6 +103,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_13_1 = new Version(2130199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_3_0_0;
 

From b09d9ef9d499119e20eacbf8ed459f929ac1b648 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Thu, 2 May 2024 13:12:59 -0400
Subject: [PATCH 04/52] Fix
 org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT.test
 {p0=search.aggregation/400_inner_hits/Include inner hits in top hits}
 (#13510)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 .../rest-api-spec/test/search.aggregation/400_inner_hits.yml   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml
index d4584a251816e..cbfc92a40835c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml
@@ -36,6 +36,9 @@ setup:
 
 ---
 "Include inner hits in top hits":
+  - skip:
+      version: " - 2.13.99"
+      reason: "the regression was fixed in 2.14.0"
   - do:
       search:
         rest_total_hits_as_int: true

From 5ff50a7e00698b0f448bc4840461c8518500d591 Mon Sep 17 00:00:00 2001
From: Martin Gaievski <gaievski@amazon.com>
Date: Thu, 2 May 2024 11:37:53 -0700
Subject: [PATCH 05/52] Refactor implementations of query phase searcher, add
 empty QueryCollectorContext (#13481)

* Refactor implementations of query hpase searcher by adding overloaded searchWith method

Signed-off-by: Martin Gaievski <gaievski@amazon.com>

* Switched to Empty context add rescoring interface

Signed-off-by: Martin Gaievski <gaievski@amazon.com>

* Changed if by simple null check for querycontext argument

Signed-off-by: Martin Gaievski <gaievski@amazon.com>

* Added Override annotation for searchWith method

Signed-off-by: Martin Gaievski <gaievski@amazon.com>

* Remove old override method from Concurrent Query Phase Searcher

Signed-off-by: Martin Gaievski <gaievski@amazon.com>

---------

Signed-off-by: Martin Gaievski <gaievski@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../query/ConcurrentQueryPhaseSearcher.java   | 26 +++++++++-----
 .../search/query/QueryCollectorContext.java   | 23 ++++++++++++
 .../opensearch/search/query/QueryPhase.java   | 36 +++++++++++++++----
 .../query/RescoringQueryCollectorContext.java | 27 ++++++++++++++
 .../search/query/TopDocsCollectorContext.java |  2 +-
 6 files changed, 100 insertions(+), 15 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5bff49af99473..ca2b174c98001 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292))
 - Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412))
 - [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
+- Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
 
 ### Deprecated
 
diff --git a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java
index e22f766d3894c..771ac60dfb5e5 100644
--- a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java
+++ b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java
@@ -23,10 +23,9 @@
 
 import java.io.IOException;
 import java.util.LinkedList;
+import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 
-import static org.opensearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext;
-
 /**
  * The implementation of the {@link QueryPhaseSearcher} which attempts to use concurrent
  * search of Apache Lucene segments if it has been enabled.
@@ -46,10 +45,19 @@ protected boolean searchWithCollector(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectors,
+        QueryCollectorContext queryCollectorContext,
         boolean hasFilterCollector,
         boolean hasTimeout
     ) throws IOException {
-        return searchWithCollectorManager(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout);
+        return searchWithCollectorManager(
+            searchContext,
+            searcher,
+            query,
+            collectors,
+            queryCollectorContext,
+            hasFilterCollector,
+            hasTimeout
+        );
     }
 
     private static boolean searchWithCollectorManager(
@@ -57,13 +65,12 @@ private static boolean searchWithCollectorManager(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectorContexts,
+        QueryCollectorContext queryCollectorContext,
         boolean hasFilterCollector,
         boolean timeoutSet
     ) throws IOException {
-        // create the top docs collector last when the other collectors are known
-        final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
-        // add the top docs collector, the first collector context in the chain
-        collectorContexts.addFirst(topDocsFactory);
+        // add the passed collector, the first collector context in the chain
+        collectorContexts.addFirst(Objects.requireNonNull(queryCollectorContext));
 
         final QuerySearchResult queryResult = searchContext.queryResult();
         final CollectorManager<?, ReduceableSearchResult> collectorManager;
@@ -95,7 +102,10 @@ private static boolean searchWithCollectorManager(
             queryResult.terminatedEarly(false);
         }
 
-        return topDocsFactory.shouldRescore();
+        if (queryCollectorContext instanceof RescoringQueryCollectorContext) {
+            return ((RescoringQueryCollectorContext) queryCollectorContext).shouldRescore();
+        }
+        return false;
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java
index 91762bee2ac08..08b048cf682bb 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java
@@ -77,6 +77,29 @@ public ScoreMode scoreMode() {
         }
     };
 
+    public static final QueryCollectorContext EMPTY_CONTEXT = new QueryCollectorContext("empty") {
+
+        @Override
+        Collector create(Collector in) throws IOException {
+            return EMPTY_COLLECTOR;
+        }
+
+        @Override
+        CollectorManager<?, ReduceableSearchResult> createManager(CollectorManager<?, ReduceableSearchResult> in) throws IOException {
+            return new CollectorManager<Collector, ReduceableSearchResult>() {
+                @Override
+                public Collector newCollector() throws IOException {
+                    return EMPTY_COLLECTOR;
+                }
+
+                @Override
+                public ReduceableSearchResult reduce(Collection<Collector> collectors) throws IOException {
+                    return result -> {};
+                }
+            };
+        }
+    };
+
     private String profilerName;
 
     QueryCollectorContext(String profilerName) {
diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
index 8f98f0d9efbd4..608649ad22b23 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
@@ -335,13 +335,12 @@ private static boolean searchWithCollector(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectors,
+        QueryCollectorContext queryCollectorContext,
         boolean hasFilterCollector,
         boolean timeoutSet
     ) throws IOException {
-        // create the top docs collector last when the other collectors are known
-        final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
-        // add the top docs collector, the first collector context in the chain
-        collectors.addFirst(topDocsFactory);
+        // add passed collector, the first collector context in the chain
+        collectors.addFirst(Objects.requireNonNull(queryCollectorContext));
 
         final Collector queryCollector;
         if (searchContext.getProfilers() != null) {
@@ -370,7 +369,10 @@ private static boolean searchWithCollector(
         for (QueryCollectorContext ctx : collectors) {
             ctx.postProcess(queryResult);
         }
-        return topDocsFactory.shouldRescore();
+        if (queryCollectorContext instanceof RescoringQueryCollectorContext) {
+            return ((RescoringQueryCollectorContext) queryCollectorContext).shouldRescore();
+        }
+        return false;
     }
 
     /**
@@ -440,7 +442,29 @@ protected boolean searchWithCollector(
             boolean hasFilterCollector,
             boolean hasTimeout
         ) throws IOException {
-            return QueryPhase.searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout);
+            // create the top docs collector last when the other collectors are known
+            final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
+            return searchWithCollector(searchContext, searcher, query, collectors, topDocsFactory, hasFilterCollector, hasTimeout);
+        }
+
+        protected boolean searchWithCollector(
+            SearchContext searchContext,
+            ContextIndexSearcher searcher,
+            Query query,
+            LinkedList<QueryCollectorContext> collectors,
+            QueryCollectorContext queryCollectorContext,
+            boolean hasFilterCollector,
+            boolean hasTimeout
+        ) throws IOException {
+            return QueryPhase.searchWithCollector(
+                searchContext,
+                searcher,
+                query,
+                collectors,
+                queryCollectorContext,
+                hasFilterCollector,
+                hasTimeout
+            );
         }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java
new file mode 100644
index 0000000000000..dc47ee80dba42
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.query;
+
+import org.opensearch.common.annotation.PublicApi;
+
+/**
+ * Abstraction that allows indication of whether results should be rescored or not based on
+ * custom logic of exact {@link QueryCollectorContext} implementation.
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "2.15.0")
+public interface RescoringQueryCollectorContext {
+
+    /**
+     * Indicates if results from the query context should be rescored
+     * @return true if results must be rescored, false otherwise
+     */
+    boolean shouldRescore();
+}
diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
index 65d3948c8401e..f780f6fe32af2 100644
--- a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
+++ b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
@@ -95,7 +95,7 @@
  *
  * @opensearch.internal
  */
-public abstract class TopDocsCollectorContext extends QueryCollectorContext {
+public abstract class TopDocsCollectorContext extends QueryCollectorContext implements RescoringQueryCollectorContext {
     protected final int numHits;
 
     TopDocsCollectorContext(String profilerName, int numHits) {

From 3b0cd0aef0097715041b65f5f0c942d1451b71d3 Mon Sep 17 00:00:00 2001
From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com>
Date: Fri, 3 May 2024 00:49:16 +0530
Subject: [PATCH 06/52] Increase cluster publish timeout in flaky test
 testStaleClusterManagerNotHijackingMajority (#13463)

Signed-off-by: Rahul Karajgikar <karajgik@amazon.com>
Co-authored-by: Rahul Karajgikar <karajgik@amazon.com>
---
 .../opensearch/discovery/StableClusterManagerDisruptionIT.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java
index f6f2b2dbfd096..4de79a31d0146 100644
--- a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java
@@ -203,7 +203,7 @@ public void testStaleClusterManagerNotHijackingMajority() throws Exception {
             3,
             Settings.builder()
                 .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
-                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s")
+                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "2s")
                 .build()
         );
         ensureStableCluster(3);

From dc77c46de2cb3dd4aac8ba1eac667466ee3d0b4e Mon Sep 17 00:00:00 2001
From: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
Date: Thu, 2 May 2024 13:15:46 -0700
Subject: [PATCH 07/52] Allow doc value search on unindexed boolean and date
 fields (#11650)

* Making fields searchable

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Adding tests for boolean fields

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Added tests for date fields

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Updating termsQuery logic

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Spotless

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Updating terms test

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Spotless

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Ensure that the points are intersecting for doc_values

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Spotless

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Adding missing javadocs

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Adding more tests for indexedValueForSearch

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Adding MatchAllDocsQuery to asserts

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fix changelog

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Simplify loop criteria + remove IndexOrDocValuesQuery

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Add some comments

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fix indendation

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fixing boolean field tests

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Revert to correct logic for termsQuery

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Update terms logic to be more succinct

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fix terms test

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Adding tests for boolean range query + fix range query to use term query inside

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Checking if upper and lower terms are valid

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Checking if upper and lower terms are valid

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fixing changelog

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

---------

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../test/search/340_doc_values_field.yml      | 381 +++++++++++++++++-
 .../index/mapper/BooleanFieldMapper.java      |  92 ++++-
 .../index/mapper/DateFieldMapper.java         |  21 +-
 .../index/mapper/BooleanFieldMapperTests.java |  25 ++
 .../index/mapper/BooleanFieldTypeTests.java   |  69 +++-
 .../index/mapper/DateFieldTypeTests.java      |   8 +-
 .../index/query/TermsQueryBuilderTests.java   |   4 +
 8 files changed, 573 insertions(+), 28 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca2b174c98001..0ac0d64558e75 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -70,6 +70,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactoring globMatch using simpleMatchWithNormalizedStrings from Regex ([#13104](https://github.com/opensearch-project/OpenSearch/pull/13104))
 - [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292))
 - Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412))
+- Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
 - [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
 - Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
index d5ece1719dc48..a133060f07c6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
@@ -46,6 +46,18 @@
                 type: ip
                 index: true
                 doc_values: true
+              boolean:
+                  type: boolean
+                  index: true
+                  doc_values: true
+              date:
+                type: date
+                index: true
+                doc_values: true
+              date_nanos:
+                type: date_nanos
+                index: true
+                doc_values: true
 
   - do:
       bulk:
@@ -53,11 +65,11 @@
         refresh: true
         body:
           - '{"index": {"_index": "test-iodvq", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-iodvq", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.987654321Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-iodvq", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.987654321Z", "date": "2024-10-29T12:12:12.987Z" }'
 
   - do:
       search:
@@ -183,6 +195,17 @@
 
   - match: {hits.total: 1}
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            term:
+              boolean: true
+
+  - match: { hits.total: 2 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -282,6 +305,17 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            terms:
+              boolean: [true, false]
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -415,6 +449,92 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              date_nanos: {
+                gte: "2018-10-29T12:12:12.123456789Z"
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              date: {
+                gte: "2018-10-29T12:12:12.987Z",
+                lte: "2020-10-29T12:12:12.987Z"
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true,
+                gte: false
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: false,
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 0 }
 ---
 "search on fields with only index enabled":
   - do:
@@ -463,6 +583,18 @@
                 type: ip
                 index: true
                 doc_values: false
+              boolean:
+                type: boolean
+                index: true
+                doc_values: false
+              date_nanos:
+                type: date_nanos
+                index: true
+                doc_values: false
+              date:
+                type: date
+                index: true
+                doc_values: false
 
   - do:
       bulk:
@@ -470,11 +602,11 @@
         refresh: true
         body:
           - '{"index": {"_index": "test-index", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-index", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-index", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }'
 
   - do:
       search:
@@ -600,6 +732,18 @@
 
   - match: {hits.total: 1}
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            term:
+              boolean: true
+
+  - match: { hits.total: 2 }
+
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -699,6 +843,17 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            terms:
+              boolean: [true, false]
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -831,6 +986,93 @@
                 lte: "192.168.0.2"
 
   - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              date_nanos: {
+                gte: "2018-10-29T12:12:12.123456789Z"
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              date: {
+                gte: "2018-10-29T12:12:12.987Z",
+                lte: "2020-10-29T12:12:12.987Z"
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true,
+                gte: false
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: false,
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 0 }
 ---
 "search on fields with only doc_values enabled":
   - skip:
@@ -883,6 +1125,18 @@
                 type: ip
                 index: false
                 doc_values: true
+              boolean:
+                type: boolean
+                index: false
+                doc_values: true
+              date_nanos:
+                type: date_nanos
+                index: false
+                doc_values: true
+              date:
+                type: date
+                index: false
+                doc_values: true
 
   - do:
       bulk:
@@ -890,11 +1144,11 @@
         refresh: true
         body:
           - '{"index": {"_index": "test-doc-values", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }'
 
   - do:
       search:
@@ -1019,6 +1273,17 @@
 
   - match: { hits.total: 1 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            term:
+              boolean: false
+
+  - match: { hits.total: 1 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -1107,6 +1372,17 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            terms:
+              boolean: [true, false]
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -1239,3 +1515,90 @@
                 lte: "192.168.0.2"
 
   - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              date_nanos: {
+                gte: "2018-10-29T12:12:12.123456789Z"
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              date: {
+                gte: "2018-10-29T12:12:12.987Z",
+                lte: "2020-10-29T12:12:12.987Z"
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true,
+                gte: false
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: false,
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 0 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
index 3c7925809415a..b4cf585c1329d 100644
--- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
@@ -37,8 +37,11 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.common.Booleans;
 import org.opensearch.common.Nullable;
@@ -55,8 +58,10 @@
 import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 
 /**
@@ -175,6 +180,10 @@ public BooleanFieldType(String name, boolean searchable) {
             this(name, searchable, false, true, false, Collections.emptyMap());
         }
 
+        public BooleanFieldType(String name, boolean searchable, boolean hasDocValues) {
+            this(name, searchable, false, hasDocValues, false, Collections.emptyMap());
+        }
+
         @Override
         public String typeName() {
             return CONTENT_TYPE;
@@ -257,16 +266,81 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
             return DocValueFormat.BOOLEAN;
         }
 
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            if (!isSearchable()) {
+                return SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0);
+            }
+            Query query = new TermQuery(new Term(name(), indexedValueForSearch(value)));
+            if (boost() != 1f) {
+                query = new BoostQuery(query, boost());
+            }
+            return query;
+        }
+
+        @Override
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            int distinct = 0;
+            Set<?> distinctValues = new HashSet<>(values);
+            for (Object value : distinctValues) {
+                if (Values.TRUE.equals(indexedValueForSearch(value))) {
+                    distinct |= 2;
+                } else if (Values.FALSE.equals(indexedValueForSearch(value))) {
+                    distinct |= 1;
+                }
+                if (distinct == 3) {
+                    return this.existsQuery(context);
+                }
+            }
+            switch (distinct) {
+                case 1:
+                    return termQuery("false", context);
+                case 2:
+                    return termQuery("true", context);
+            }
+
+            return new MatchNoDocsQuery("Values did not contain True or False");
+        }
+
         @Override
         public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
-            failIfNotIndexed();
-            return new TermRangeQuery(
-                name(),
-                lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
-                upperTerm == null ? null : indexedValueForSearch(upperTerm),
-                includeLower,
-                includeUpper
-            );
+            failIfNotIndexedAndNoDocValues();
+            if (lowerTerm == null) {
+                lowerTerm = false;
+                includeLower = true;
+
+            }
+            if (upperTerm == null) {
+                upperTerm = true;
+                includeUpper = true;
+
+            }
+
+            lowerTerm = indexedValueForSearch(lowerTerm);
+            upperTerm = indexedValueForSearch(upperTerm);
+
+            if (lowerTerm == upperTerm) {
+                if (!includeLower || !includeUpper) {
+                    return new MatchNoDocsQuery();
+                }
+                return termQuery(lowerTerm.equals(Values.TRUE), context);
+            }
+
+            if (lowerTerm.equals(Values.TRUE)) {
+                return new MatchNoDocsQuery();
+            }
+            if (!includeLower && !includeUpper) {
+                return new MatchNoDocsQuery();
+            } else if (!includeLower) {
+                return termQuery(true, context);
+            } else if (!includeUpper) {
+                return termQuery(false, context);
+            } else {
+                return this.existsQuery(context);
+            }
+
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
index d98e6ea6af83d..b7ee3bb8ca3e3 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
@@ -457,22 +457,30 @@ public Query rangeQuery(
             @Nullable DateMathParser forcedDateParser,
             QueryShardContext context
         ) {
-            failIfNotIndexed();
+            failIfNotIndexedAndNoDocValues();
             if (relation == ShapeRelation.DISJOINT) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges");
             }
             DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser;
             return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
-                Query query = LongPoint.newRangeQuery(name(), l, u);
-                if (hasDocValues()) {
+                if (isSearchable() && hasDocValues()) {
+                    Query query = LongPoint.newRangeQuery(name(), l, u);
                     Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
                     query = new IndexOrDocValuesQuery(query, dvQuery);
 
                     if (context.indexSortedOnField(name())) {
                         query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
                     }
+                    return query;
+                }
+                if (hasDocValues()) {
+                    Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
+                    if (context.indexSortedOnField(name())) {
+                        query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
+                    }
+                    return query;
                 }
-                return query;
+                return LongPoint.newRangeQuery(name(), l, u);
             });
         }
 
@@ -543,6 +551,7 @@ public static long parseToLong(
 
         @Override
         public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
             long originLong = parseToLong(origin, true, null, null, context::nowInMillis);
             TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot");
             return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime);
@@ -559,6 +568,10 @@ public Relation isFieldWithinQuery(
             DateMathParser dateParser,
             QueryRewriteContext context
         ) throws IOException {
+            // if we have only doc_values enabled we do not look at the BKD so we return an INTERSECTS by default
+            if (isSearchable() == false && hasDocValues()) {
+                return Relation.INTERSECTS;
+            }
             if (dateParser == null) {
                 dateParser = this.dateMathParser;
             }
diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
index 8dec03a353d16..5392bd6c358d3 100644
--- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
@@ -209,4 +209,29 @@ public void testBoosts() throws Exception {
         assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null));
         assertParseMaximalWarnings();
     }
+
+    public void testIndexedValueForSearch() throws Exception {
+        assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(null), BooleanFieldMapper.Values.FALSE);
+
+        assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(false), BooleanFieldMapper.Values.FALSE);
+
+        assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(true), BooleanFieldMapper.Values.TRUE);
+
+        assertEquals(
+            new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("true")),
+            BooleanFieldMapper.Values.TRUE
+        );
+
+        assertEquals(
+            new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("false")),
+            BooleanFieldMapper.Values.FALSE
+        );
+
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("random"))
+        );
+
+        assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
index 14092706411cb..aab63fc30efd7 100644
--- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
@@ -31,11 +31,18 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 public class BooleanFieldTypeTests extends FieldTypeTestCase {
 
@@ -59,9 +66,67 @@ public void testTermQuery() {
         assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", null));
         assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", null));
 
-        MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false);
+        MappedFieldType doc_ft = new BooleanFieldMapper.BooleanFieldType("field", false, true);
+        assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), doc_ft.termQuery("true", null));
+        assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 0), doc_ft.termQuery("false", null));
+
+        MappedFieldType boost_ft = new BooleanFieldMapper.BooleanFieldType("field");
+        boost_ft.setBoost(2f);
+        assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2f), boost_ft.termQuery("true", null));
+        assertEquals(new BoostQuery(new TermQuery(new Term("field", "F")), 2f), boost_ft.termQuery("false", null));
+
+        MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", null));
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
+    }
+
+    public void testTermsQuery() {
+        MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field");
+        List<BytesRef> terms = new ArrayList<>();
+        terms.add(new BytesRef("true"));
+        terms.add(new BytesRef("false"));
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.termsQuery(terms, null));
+
+        List<BytesRef> newTerms = new ArrayList<>();
+        newTerms.add(new BytesRef("true"));
+        assertEquals(new TermQuery(new Term("field", "T")), ft.termsQuery(newTerms, null));
+
+        List<BytesRef> incorrectTerms = new ArrayList<>();
+        incorrectTerms.add(new BytesRef("true"));
+        incorrectTerms.add(new BytesRef("random"));
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ft.termsQuery(incorrectTerms, null));
+        assertEquals("Can't parse boolean value [random], expected [true] or [false]", ex.getMessage());
+
+        MappedFieldType doc_only_ft = new BooleanFieldMapper.BooleanFieldType("field", false, true);
+
+        assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), doc_only_ft.termsQuery(newTerms, null));
+
+        MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(terms, null));
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
+    }
+
+    public void testRangeQuery() {
+        BooleanFieldMapper.BooleanFieldType ft = new BooleanFieldMapper.BooleanFieldType("field");
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.rangeQuery(false, true, true, true, null));
+
+        assertEquals(new TermQuery(new Term("field", "T")), ft.rangeQuery(false, true, false, true, null));
+
+        assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(false, true, true, false, null));
+
+        assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(false, true, false, false, null));
+
+        assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(false, true, false, false, null));
+
+        assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(false, false, true, true, null));
+
+        assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(null, false, true, true, null));
+
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.rangeQuery(false, null, true, true, null));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.rangeQuery("random", null, true, true, null));
+
+        assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage());
     }
 
     public void testFetchSourceValue() throws IOException {
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
index ab53ae81ab0ce..db5e1e419de93 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
@@ -216,14 +216,14 @@ public void testTermQuery() {
             "field",
             false,
             false,
-            true,
+            false,
             DateFieldMapper.getDefaultDateTimeFormatter(),
             Resolution.MILLISECONDS,
             null,
             Collections.emptyMap()
         );
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context));
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
     }
 
     public void testRangeQuery() throws IOException {
@@ -279,7 +279,7 @@ public void testRangeQuery() throws IOException {
             "field",
             false,
             false,
-            true,
+            false,
             DateFieldMapper.getDefaultDateTimeFormatter(),
             Resolution.MILLISECONDS,
             null,
@@ -289,7 +289,7 @@ public void testRangeQuery() throws IOException {
             IllegalArgumentException.class,
             () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context)
         );
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
     }
 
     public void testRangeQueryWithIndexSort() {
diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
index 32bf290627b63..61f7dc5facebe 100644
--- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
@@ -34,7 +34,9 @@
 
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PointInSetQuery;
 import org.apache.lucene.search.Query;
@@ -137,6 +139,8 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query,
                     .or(instanceOf(ConstantScoreQuery.class))
                     .or(instanceOf(MatchNoDocsQuery.class))
                     .or(instanceOf(IndexOrDocValuesQuery.class))
+                    .or(instanceOf(MatchAllDocsQuery.class))
+                    .or(instanceOf(FieldExistsQuery.class))
             );
             if (query instanceof ConstantScoreQuery) {
                 assertThat(((ConstantScoreQuery) query).getQuery(), instanceOf(BooleanQuery.class));

From 79957ebc03c4af746a31cdeb47158bad96553505 Mon Sep 17 00:00:00 2001
From: Chenyang Ji <cyji@amazon.com>
Date: Thu, 2 May 2024 14:48:23 -0700
Subject: [PATCH 08/52] Add support for deep copying SearchRequest (#12295)

Signed-off-by: Chenyang Ji <cyji@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../action/search/SearchRequest.java          | 13 ++++++++
 .../action/search/SearchRequestTests.java     | 30 +++++++++++++++++++
 3 files changed, 44 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ac0d64558e75..ba55a80f5d1d6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225))
 - [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))
 - [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276))
+- Add support for deep copying SearchRequest ([#12295](https://github.com/opensearch-project/OpenSearch/pull/12295))
 
 ### Dependencies
 - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
index f3d9f77e2394c..4d3bb868b779a 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -161,6 +162,18 @@ public SearchRequest(String[] indices, SearchSourceBuilder source) {
         this.source = source;
     }
 
+    /**
+     * Deep clone a SearchRequest
+     *
+     * @return a copy of the current SearchRequest
+     */
+    public SearchRequest deepCopy() throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        this.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        return new SearchRequest(in);
+    }
+
     /**
      * Creates a new sub-search request starting from the original search request that is provided.
      * For internal use only, allows to fork a search request into multiple search requests that will be executed independently.
diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
index 9ee314e77ca7e..40514c526f190 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
@@ -46,6 +46,7 @@
 import org.opensearch.search.Scroll;
 import org.opensearch.search.builder.PointInTimeBuilder;
 import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.fetch.subphase.FetchSourceContext;
 import org.opensearch.search.rescore.QueryRescorerBuilder;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
@@ -76,6 +77,35 @@ protected SearchRequest createSearchRequest() throws IOException {
         );
     }
 
+    public void testClone() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest clonedRequest = searchRequest.deepCopy();
+        assertEquals(searchRequest.hashCode(), clonedRequest.hashCode());
+        assertNotSame(searchRequest, clonedRequest);
+
+        String[] includes = new String[] { "field1.*" };
+        String[] excludes = new String[] { "field2.*" };
+        FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+        SearchSourceBuilder source = new SearchSourceBuilder().fetchSource(fetchSourceContext);
+        SearchRequest complexSearchRequest = createSearchRequest().source(source);
+        complexSearchRequest.requestCache(false);
+        complexSearchRequest.scroll(new TimeValue(1000));
+        SearchRequest clonedComplexRequest = complexSearchRequest.deepCopy();
+        assertEquals(complexSearchRequest.hashCode(), clonedComplexRequest.hashCode());
+        assertNotSame(complexSearchRequest, clonedComplexRequest);
+        assertEquals(fetchSourceContext, clonedComplexRequest.source().fetchSource());
+        assertNotSame(fetchSourceContext, clonedComplexRequest.source().fetchSource());
+        // Change the value of the original includes array and excludes array
+        includes[0] = "new_field1.*";
+        excludes[0] = "new_field2.*";
+        // Values in the original fetchSource object should be updated
+        assertEquals("new_field1.*", complexSearchRequest.source().fetchSource().includes()[0]);
+        assertEquals("new_field2.*", complexSearchRequest.source().fetchSource().excludes()[0]);
+        // Values in the cloned fetchSource object should not be updated
+        assertEquals("field1.*", clonedComplexRequest.source().fetchSource().includes()[0]);
+        assertEquals("field2.*", clonedComplexRequest.source().fetchSource().excludes()[0]);
+    }
+
     public void testWithLocalReduction() {
         expectThrows(NullPointerException.class, () -> SearchRequest.subSearchRequest(null, Strings.EMPTY_ARRAY, "", 0, randomBoolean()));
         SearchRequest request = new SearchRequest();

From 37df569367c5f57ee80785dbd446bd7ed7e49faa Mon Sep 17 00:00:00 2001
From: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
Date: Thu, 2 May 2024 15:57:33 -0700
Subject: [PATCH 09/52] Update TermsQueryBuilderTests to include TermQuery
 (#13517)

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
---
 .../java/org/opensearch/index/query/TermsQueryBuilderTests.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
index 61f7dc5facebe..97f372dc04a1b 100644
--- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
@@ -41,6 +41,7 @@
 import org.apache.lucene.search.PointInSetQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.get.GetRequest;
@@ -141,6 +142,7 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query,
                     .or(instanceOf(IndexOrDocValuesQuery.class))
                     .or(instanceOf(MatchAllDocsQuery.class))
                     .or(instanceOf(FieldExistsQuery.class))
+                    .or(instanceOf(TermQuery.class))
             );
             if (query instanceof ConstantScoreQuery) {
                 assertThat(((ConstantScoreQuery) query).getQuery(), instanceOf(BooleanQuery.class));

From a8017d84c45c5a32fbb35f84770ce4963cea8b1f Mon Sep 17 00:00:00 2001
From: Peter Alfonsi <peter.alfonsi@gmail.com>
Date: Thu, 2 May 2024 17:29:07 -0700
Subject: [PATCH 10/52] [Tiered Caching] Adds stats implementation for
 TieredSpilloverCache (#13236)

Stats rework part 4 of 4
---------

Signed-off-by: Peter Alfonsi <petealft@amazon.com>
Co-authored-by: Peter Alfonsi <petealft@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../TieredSpilloverCacheIT.java               |   2 +-
 .../common/tier/TieredSpilloverCache.java     | 250 ++++++++++++----
 .../tier/TieredSpilloverCacheStatsHolder.java | 161 +++++++++++
 .../cache/common/tier/MockDiskCache.java      |  27 +-
 .../tier/TieredSpilloverCacheTests.java       | 266 ++++++++++++++++--
 .../cache/store/disk/EhcacheDiskCache.java    |  11 +-
 .../store/disk/EhCacheDiskCacheTests.java     |  32 +++
 .../cache/stats/DefaultCacheStatsHolder.java  |  30 +-
 .../cache/store/OpenSearchOnHeapCache.java    |  18 +-
 .../cache/store/builders/ICacheBuilder.java   |  11 +
 .../cache/store/config/CacheConfig.java       |  13 +
 .../store/OpenSearchOnHeapCacheTests.java     |  33 ++-
 13 files changed, 755 insertions(+), 100 deletions(-)
 create mode 100644 modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba55a80f5d1d6..cdaa1904e2522 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Tiered Caching] Add dimension-based stats to ICache implementations. ([#12531](https://github.com/opensearch-project/OpenSearch/pull/12531))
 - Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
 - Add an individual setting of rate limiter for segment replication ([#12959](https://github.com/opensearch-project/OpenSearch/pull/12959))
+- [Tiered Caching] Add dimension-based stats to TieredSpilloverCache ([#13236](https://github.com/opensearch-project/OpenSearch/pull/13236))
 - [Tiered Caching] Expose new cache stats API ([#13237](https://github.com/opensearch-project/OpenSearch/pull/13237))
 - [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
 - Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
diff --git a/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java b/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java
index cbe16a690c104..bfc184cff0566 100644
--- a/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java
+++ b/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java
@@ -550,7 +550,7 @@ public MockDiskCachePlugin() {}
 
         @Override
         public Map<String, ICache.Factory> getCacheFactoryMap() {
-            return Map.of(MockDiskCache.MockDiskCacheFactory.NAME, new MockDiskCache.MockDiskCacheFactory(0, 1000));
+            return Map.of(MockDiskCache.MockDiskCacheFactory.NAME, new MockDiskCache.MockDiskCacheFactory(0, 1000, false));
         }
 
         @Override
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
index bca81ebd958ce..9942651ccdd67 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
@@ -20,6 +20,7 @@
 import org.opensearch.common.cache.policy.CachedQueryResult;
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.config.CacheConfig;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -34,12 +35,16 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.function.ToLongBiFunction;
 
 import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP;
 
 /**
  * This cache spillover the evicted items from heap tier to disk tier. All the new items are first cached on heap
@@ -60,9 +65,17 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
     private final ICache<K, V> diskCache;
     private final ICache<K, V> onHeapCache;
 
-    // The listener for removals from the spillover cache as a whole
-    // TODO: In TSC stats PR, each tier will have its own separate removal listener.
+    // Removal listeners for the individual tiers
+    private final RemovalListener<ICacheKey<K>, V> onDiskRemovalListener;
+    private final RemovalListener<ICacheKey<K>, V> onHeapRemovalListener;
+
+    // Removal listener from the spillover cache as a whole
     private final RemovalListener<ICacheKey<K>, V> removalListener;
+
+    // In future we want to just read the stats from the individual tiers' statsHolder objects, but this isn't
+    // possible right now because of the way computeIfAbsent is implemented.
+    private final TieredSpilloverCacheStatsHolder statsHolder;
+    private ToLongBiFunction<ICacheKey<K>, V> weigher;
     private final List<String> dimensionNames;
     ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
     ReleasableLock readLock = new ReleasableLock(readWriteLock.readLock());
@@ -70,7 +83,7 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
     /**
      * Maintains caching tiers in ascending order of cache latency.
      */
-    private final Map<ICache<K, V>, Boolean> caches;
+    private final Map<ICache<K, V>, TierInfo> caches;
     private final List<Predicate<V>> policies;
 
     TieredSpilloverCache(Builder<K, V> builder) {
@@ -80,21 +93,12 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
         Objects.requireNonNull(builder.cacheConfig.getClusterSettings(), "cluster settings can't be null");
         this.removalListener = Objects.requireNonNull(builder.removalListener, "Removal listener can't be null");
 
+        this.onHeapRemovalListener = new HeapTierRemovalListener(this);
+        this.onDiskRemovalListener = new DiskTierRemovalListener(this);
+        this.weigher = Objects.requireNonNull(builder.cacheConfig.getWeigher(), "Weigher can't be null");
+
         this.onHeapCache = builder.onHeapCacheFactory.create(
-            new CacheConfig.Builder<K, V>().setRemovalListener(new RemovalListener<ICacheKey<K>, V>() {
-                @Override
-                public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
-                    try (ReleasableLock ignore = writeLock.acquire()) {
-                        if (caches.get(diskCache)
-                            && SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason())
-                            && evaluatePolicies(notification.getValue())) {
-                            diskCache.put(notification.getKey(), notification.getValue());
-                        } else {
-                            removalListener.onRemoval(notification);
-                        }
-                    }
-                }
-            })
+            new CacheConfig.Builder<K, V>().setRemovalListener(onHeapRemovalListener)
                 .setKeyType(builder.cacheConfig.getKeyType())
                 .setValueType(builder.cacheConfig.getValueType())
                 .setSettings(builder.cacheConfig.getSettings())
@@ -103,18 +107,33 @@ && evaluatePolicies(notification.getValue())) {
                 .setMaxSizeInBytes(builder.cacheConfig.getMaxSizeInBytes())
                 .setExpireAfterAccess(builder.cacheConfig.getExpireAfterAccess())
                 .setClusterSettings(builder.cacheConfig.getClusterSettings())
+                .setStatsTrackingEnabled(false)
                 .build(),
             builder.cacheType,
             builder.cacheFactories
 
         );
-        this.diskCache = builder.diskCacheFactory.create(builder.cacheConfig, builder.cacheType, builder.cacheFactories);
+        this.diskCache = builder.diskCacheFactory.create(
+            new CacheConfig.Builder<K, V>().setRemovalListener(onDiskRemovalListener)
+                .setKeyType(builder.cacheConfig.getKeyType())
+                .setValueType(builder.cacheConfig.getValueType())
+                .setSettings(builder.cacheConfig.getSettings())
+                .setWeigher(builder.cacheConfig.getWeigher())
+                .setDimensionNames(builder.cacheConfig.getDimensionNames())
+                .setStatsTrackingEnabled(false)
+                .build(),
+            builder.cacheType,
+            builder.cacheFactories
+        );
         Boolean isDiskCacheEnabled = DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType).get(builder.cacheConfig.getSettings());
-        LinkedHashMap<ICache<K, V>, Boolean> cacheListMap = new LinkedHashMap<>();
-        cacheListMap.put(onHeapCache, true);
-        cacheListMap.put(diskCache, isDiskCacheEnabled);
+        LinkedHashMap<ICache<K, V>, TierInfo> cacheListMap = new LinkedHashMap<>();
+        cacheListMap.put(onHeapCache, new TierInfo(true, TIER_DIMENSION_VALUE_ON_HEAP));
+        cacheListMap.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK));
         this.caches = Collections.synchronizedMap(cacheListMap);
+
         this.dimensionNames = builder.cacheConfig.getDimensionNames();
+        // Pass "tier" as the innermost dimension name, in addition to whatever dimensions are specified for the cache as a whole
+        this.statsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, isDiskCacheEnabled);
         this.policies = builder.policies; // Will never be null; builder initializes it to an empty list
         builder.cacheConfig.getClusterSettings()
             .addSettingsUpdateConsumer(DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType), this::enableDisableDiskCache);
@@ -134,25 +153,38 @@ ICache<K, V> getDiskCache() {
     void enableDisableDiskCache(Boolean isDiskCacheEnabled) {
         // When disk cache is disabled, we are not clearing up the disk cache entries yet as that should be part of
         // separate cache/clear API.
-        this.caches.put(diskCache, isDiskCacheEnabled);
+        this.caches.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK));
+        this.statsHolder.setDiskCacheEnabled(isDiskCacheEnabled);
     }
 
     @Override
     public V get(ICacheKey<K> key) {
-        return getValueFromTieredCache().apply(key);
+        Tuple<V, String> cacheValueTuple = getValueFromTieredCache(true).apply(key);
+        if (cacheValueTuple == null) {
+            return null;
+        }
+        return cacheValueTuple.v1();
     }
 
     @Override
     public void put(ICacheKey<K> key, V value) {
         try (ReleasableLock ignore = writeLock.acquire()) {
             onHeapCache.put(key, value);
+            updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value);
         }
     }
 
     @Override
     public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
-        V cacheValue = getValueFromTieredCache().apply(key);
-        if (cacheValue == null) {
+        // Don't capture stats in the initial getValueFromTieredCache(). If we have concurrent requests for the same key,
+        // and it only has to be loaded one time, we should report one miss and the rest hits. But, if we do stats in
+        // getValueFromTieredCache(),
+        // we will see all misses. Instead, handle stats in computeIfAbsent().
+        Tuple<V, String> cacheValueTuple = getValueFromTieredCache(false).apply(key);
+        List<String> heapDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_ON_HEAP);
+        List<String> diskDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_DISK);
+
+        if (cacheValueTuple == null) {
             // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside.
             // This is needed as there can be many requests for the same key at the same time and we only want to load
             // the value once.
@@ -160,19 +192,49 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
             try (ReleasableLock ignore = writeLock.acquire()) {
                 value = onHeapCache.computeIfAbsent(key, loader);
             }
+            // Handle stats
+            if (loader.isLoaded()) {
+                // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk cache
+                // if present
+                updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value);
+                statsHolder.incrementMisses(heapDimensionValues);
+                if (caches.get(diskCache).isEnabled()) {
+                    statsHolder.incrementMisses(diskDimensionValues);
+                }
+            } else {
+                // Another thread requesting this key already loaded the value. Register a hit for the heap cache
+                statsHolder.incrementHits(heapDimensionValues);
+            }
             return value;
+        } else {
+            // Handle stats for an initial hit from getValueFromTieredCache()
+            if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_ON_HEAP)) {
+                // A hit for the heap tier
+                statsHolder.incrementHits(heapDimensionValues);
+            } else if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_DISK)) {
+                // Miss for the heap tier, hit for the disk tier
+                statsHolder.incrementMisses(heapDimensionValues);
+                statsHolder.incrementHits(diskDimensionValues);
+            }
         }
-        return cacheValue;
+        return cacheValueTuple.v1();
     }
 
     @Override
     public void invalidate(ICacheKey<K> key) {
         // We are trying to invalidate the key from all caches though it would be present in only of them.
         // Doing this as we don't know where it is located. We could do a get from both and check that, but what will
-        // also count hits/misses stats, so ignoring it for now.
-        try (ReleasableLock ignore = writeLock.acquire()) {
-            for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
-                cacheEntry.getKey().invalidate(key);
+        // also trigger a hit/miss listener event, so ignoring it for now.
+        // We don't update stats here, as this is handled by the removal listeners for the tiers.
+        for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+            if (key.getDropStatsForDimensions()) {
+                List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, cacheEntry.getValue().tierName);
+                statsHolder.removeDimensions(dimensionValues);
+            }
+            if (key.key != null) {
+                try (ReleasableLock ignore = writeLock.acquire()) {
+                    cacheEntry.getKey().invalidate(key);
+                }
             }
         }
     }
@@ -180,10 +242,11 @@ public void invalidate(ICacheKey<K> key) {
     @Override
     public void invalidateAll() {
         try (ReleasableLock ignore = writeLock.acquire()) {
-            for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
+            for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
                 cacheEntry.getKey().invalidateAll();
             }
         }
+        statsHolder.reset();
     }
 
     /**
@@ -194,7 +257,7 @@ public void invalidateAll() {
     @Override
     public Iterable<ICacheKey<K>> keys() {
         List<Iterable<ICacheKey<K>>> iterableList = new ArrayList<>();
-        for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
+        for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
             iterableList.add(cacheEntry.getKey().keys());
         }
         Iterable<ICacheKey<K>>[] iterables = (Iterable<ICacheKey<K>>[]) iterableList.toArray(new Iterable<?>[0]);
@@ -203,19 +266,15 @@ public Iterable<ICacheKey<K>> keys() {
 
     @Override
     public long count() {
-        long count = 0;
-        for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
-            // Count for all the tiers irrespective of whether they are enabled or not. As eventually
-            // this will turn to zero once cache is cleared up either via invalidation or manually.
-            count += cacheEntry.getKey().count();
-        }
-        return count;
+        // Count for all the tiers irrespective of whether they are enabled or not. As eventually
+        // this will turn to zero once cache is cleared up either via invalidation or manually.
+        return statsHolder.count();
     }
 
     @Override
     public void refresh() {
         try (ReleasableLock ignore = writeLock.acquire()) {
-            for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
+            for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
                 cacheEntry.getKey().refresh();
             }
         }
@@ -223,7 +282,7 @@ public void refresh() {
 
     @Override
     public void close() throws IOException {
-        for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
+        for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
             // Close all the caches here irrespective of whether they are enabled or not.
             cacheEntry.getKey().close();
         }
@@ -231,25 +290,76 @@ public void close() throws IOException {
 
     @Override
     public ImmutableCacheStatsHolder stats(String[] levels) {
-        return null; // TODO: in TSC stats PR
+        return statsHolder.getImmutableCacheStatsHolder(levels);
     }
 
-    private Function<ICacheKey<K>, V> getValueFromTieredCache() {
+    /**
+     * Get a value from the tiered cache, and the name of the tier it was found in.
+     * @param captureStats Whether to record hits/misses for this call of the function
+     * @return A tuple of the value and the name of the tier it was found in.
+     */
+    private Function<ICacheKey<K>, Tuple<V, String>> getValueFromTieredCache(boolean captureStats) {
         return key -> {
             try (ReleasableLock ignore = readLock.acquire()) {
-                for (Map.Entry<ICache<K, V>, Boolean> cacheEntry : caches.entrySet()) {
-                    if (cacheEntry.getValue()) {
+                for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+                    if (cacheEntry.getValue().isEnabled()) {
                         V value = cacheEntry.getKey().get(key);
+                        // Get the tier value corresponding to this cache
+                        String tierValue = cacheEntry.getValue().tierName;
+                        List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, tierValue);
                         if (value != null) {
-                            return value;
+                            if (captureStats) {
+                                statsHolder.incrementHits(dimensionValues);
+                            }
+                            return new Tuple<>(value, tierValue);
+                        } else if (captureStats) {
+                            statsHolder.incrementMisses(dimensionValues);
                         }
                     }
                 }
+                return null;
             }
-            return null;
         };
     }
 
+    void handleRemovalFromHeapTier(RemovalNotification<ICacheKey<K>, V> notification) {
+        ICacheKey<K> key = notification.getKey();
+        boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason());
+        if (caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue())) {
+            try (ReleasableLock ignore = writeLock.acquire()) {
+                diskCache.put(key, notification.getValue()); // spill over to the disk tier and increment its stats
+            }
+            updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue());
+        } else {
+            // If the value is not going to the disk cache, send this notification to the TSC's removal listener
+            // as the value is leaving the TSC entirely
+            removalListener.onRemoval(notification);
+        }
+        updateStatsOnRemoval(TIER_DIMENSION_VALUE_ON_HEAP, wasEvicted, key, notification.getValue());
+    }
+
+    void handleRemovalFromDiskTier(RemovalNotification<ICacheKey<K>, V> notification) {
+        // Values removed from the disk tier leave the TSC entirely
+        removalListener.onRemoval(notification);
+        boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason());
+        updateStatsOnRemoval(TIER_DIMENSION_VALUE_DISK, wasEvicted, notification.getKey(), notification.getValue());
+    }
+
+    void updateStatsOnRemoval(String removedFromTierValue, boolean wasEvicted, ICacheKey<K> key, V value) {
+        List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, removedFromTierValue);
+        if (wasEvicted) {
+            statsHolder.incrementEvictions(dimensionValues);
+        }
+        statsHolder.decrementItems(dimensionValues);
+        statsHolder.decrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value));
+    }
+
+    void updateStatsOnPut(String destinationTierValue, ICacheKey<K> key, V value) {
+        List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, destinationTierValue);
+        statsHolder.incrementItems(dimensionValues);
+        statsHolder.incrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value));
+    }
+
     boolean evaluatePolicies(V value) {
         for (Predicate<V> policy : policies) {
             if (!policy.test(value)) {
@@ -259,6 +369,38 @@ boolean evaluatePolicies(V value) {
         return true;
     }
 
+    /**
+     * A class which receives removal events from the heap tier.
+     */
+    private class HeapTierRemovalListener implements RemovalListener<ICacheKey<K>, V> {
+        private final TieredSpilloverCache<K, V> tsc;
+
+        HeapTierRemovalListener(TieredSpilloverCache<K, V> tsc) {
+            this.tsc = tsc;
+        }
+
+        @Override
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+            tsc.handleRemovalFromHeapTier(notification);
+        }
+    }
+
+    /**
+     * A class which receives removal events from the disk tier.
+     */
+    private class DiskTierRemovalListener implements RemovalListener<ICacheKey<K>, V> {
+        private final TieredSpilloverCache<K, V> tsc;
+
+        DiskTierRemovalListener(TieredSpilloverCache<K, V> tsc) {
+            this.tsc = tsc;
+        }
+
+        @Override
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+            tsc.handleRemovalFromDiskTier(notification);
+        }
+    }
+
     /**
      * ConcatenatedIterables which combines cache iterables and supports remove() functionality as well if underlying
      * iterator supports it.
@@ -320,6 +462,20 @@ public void remove() {
         }
     }
 
+    private class TierInfo {
+        AtomicBoolean isEnabled;
+        final String tierName;
+
+        TierInfo(boolean isEnabled, String tierName) {
+            this.isEnabled = new AtomicBoolean(isEnabled);
+            this.tierName = tierName;
+        }
+
+        boolean isEnabled() {
+            return isEnabled.get();
+        }
+    }
+
     /**
      * Factory to create TieredSpilloverCache objects.
      */
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java
new file mode 100644
index 0000000000000..d17059e8dee94
--- /dev/null
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java
@@ -0,0 +1,161 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cache.common.tier;
+
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * A tier-aware version of DefaultCacheStatsHolder. Overrides the incrementer functions, as we can't just add the on-heap
+ * and disk stats to get a total for the cache as a whole. If the disk tier is present, the total hits, size, and entries
+ * should be the sum of both tiers' values, but the total misses and evictions should be the disk tier's values.
+ * When the disk tier isn't present, on-heap misses and evictions should contribute to the total.
+ *
+ * For example, if the heap tier has 5 misses and the disk tier has 4, the total cache has had 4 misses, not 9.
+ * The same goes for evictions. Other stats values add normally.
+ *
+ * This means for misses and evictions, if we are incrementing for the on-heap tier and the disk tier is present,
+ * we have to increment only the leaf nodes corresponding to the on-heap tier itself, and not its ancestors,
+ * which correspond to totals including both tiers. If the disk tier is not present, we do increment the ancestor nodes.
+ */
+public class TieredSpilloverCacheStatsHolder extends DefaultCacheStatsHolder {
+
+    /** Whether the disk cache is currently enabled. */
+    private boolean diskCacheEnabled;
+
+    // Common values used for tier dimension
+
+    /** The name for the tier dimension. */
+    public static final String TIER_DIMENSION_NAME = "tier";
+
+    /** Dimension value for on-heap cache, like OpenSearchOnHeapCache.*/
+    public static final String TIER_DIMENSION_VALUE_ON_HEAP = "on_heap";
+
+    /** Dimension value for on-disk cache, like EhcacheDiskCache. */
+    public static final String TIER_DIMENSION_VALUE_DISK = "disk";
+
+    /**
+     * Constructor for the stats holder.
+     * @param originalDimensionNames the original dimension names, not including TIER_DIMENSION_NAME
+     * @param diskCacheEnabled whether the disk tier starts out enabled
+     */
+    public TieredSpilloverCacheStatsHolder(List<String> originalDimensionNames, boolean diskCacheEnabled) {
+        super(
+            getDimensionNamesWithTier(originalDimensionNames),
+            TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME
+        );
+        this.diskCacheEnabled = diskCacheEnabled;
+    }
+
+    private static List<String> getDimensionNamesWithTier(List<String> dimensionNames) {
+        List<String> dimensionNamesWithTier = new ArrayList<>(dimensionNames);
+        dimensionNamesWithTier.add(TIER_DIMENSION_NAME);
+        return dimensionNamesWithTier;
+    }
+
+    /**
+     * Add tierValue to the end of a copy of the initial dimension values, so they can appropriately be used in this stats holder.
+     */
+    List<String> getDimensionsWithTierValue(List<String> initialDimensions, String tierValue) {
+        List<String> result = new ArrayList<>(initialDimensions);
+        result.add(tierValue);
+        return result;
+    }
+
+    private String validateTierDimensionValue(List<String> dimensionValues) {
+        String tierDimensionValue = dimensionValues.get(dimensionValues.size() - 1);
+        assert tierDimensionValue.equals(TIER_DIMENSION_VALUE_ON_HEAP) || tierDimensionValue.equals(TIER_DIMENSION_VALUE_DISK)
+            : "Invalid tier dimension value";
+        return tierDimensionValue;
+    }
+
+    @Override
+    public void incrementHits(List<String> dimensionValues) {
+        validateTierDimensionValue(dimensionValues);
+        // Hits from either tier should be included in the total values.
+        super.incrementHits(dimensionValues);
+    }
+
+    @Override
+    public void incrementMisses(List<String> dimensionValues) {
+        final String tierValue = validateTierDimensionValue(dimensionValues);
+
+        // If the disk tier is present, only misses from the disk tier should be included in total values.
+        Consumer<Node> missIncrementer = (node) -> {
+            if (tierValue.equals(TIER_DIMENSION_VALUE_ON_HEAP) && diskCacheEnabled) {
+                // If on-heap tier, increment only the leaf node corresponding to the on heap values; not the total values in its parent
+                // nodes
+                if (node.isAtLowestLevel()) {
+                    node.incrementMisses();
+                }
+            } else {
+                // If disk tier, or on-heap tier with a disabled disk tier, increment the leaf node and its parents
+                node.incrementMisses();
+            }
+        };
+        internalIncrement(dimensionValues, missIncrementer, true);
+    }
+
+    @Override
+    public void incrementEvictions(List<String> dimensionValues) {
+        final String tierValue = validateTierDimensionValue(dimensionValues);
+
+        // If the disk tier is present, only evictions from the disk tier should be included in total values.
+        Consumer<DefaultCacheStatsHolder.Node> evictionsIncrementer = (node) -> {
+            if (tierValue.equals(TIER_DIMENSION_VALUE_ON_HEAP) && diskCacheEnabled) {
+                // If on-heap tier, increment only the leaf node corresponding to the on heap values; not the total values in its parent
+                // nodes
+                if (node.isAtLowestLevel()) {
+                    node.incrementEvictions();
+                }
+            } else {
+                // If disk tier, or on-heap tier with a disabled disk tier, increment the leaf node and its parents
+                node.incrementEvictions();
+            }
+        };
+        internalIncrement(dimensionValues, evictionsIncrementer, true);
+    }
+
+    @Override
+    public void incrementSizeInBytes(List<String> dimensionValues, long amountBytes) {
+        validateTierDimensionValue(dimensionValues);
+        // Size from either tier should be included in the total values.
+        super.incrementSizeInBytes(dimensionValues, amountBytes);
+    }
+
+    // For decrements, we should not create nodes if they are absent. This protects us from erroneously decrementing values for keys
+    // which have been entirely deleted, for example in an async removal listener.
+    @Override
+    public void decrementSizeInBytes(List<String> dimensionValues, long amountBytes) {
+        validateTierDimensionValue(dimensionValues);
+        // Size from either tier should be included in the total values.
+        super.decrementSizeInBytes(dimensionValues, amountBytes);
+    }
+
+    @Override
+    public void incrementItems(List<String> dimensionValues) {
+        validateTierDimensionValue(dimensionValues);
+        // Entries from either tier should be included in the total values.
+        super.incrementItems(dimensionValues);
+    }
+
+    @Override
+    public void decrementItems(List<String> dimensionValues) {
+        validateTierDimensionValue(dimensionValues);
+        // Entries from either tier should be included in the total values.
+        super.decrementItems(dimensionValues);
+    }
+
+    void setDiskCacheEnabled(boolean diskCacheEnabled) {
+        this.diskCacheEnabled = diskCacheEnabled;
+    }
+}
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java
index 8aed3f004e7b2..2058faa5181b1 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java
@@ -16,11 +16,15 @@
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.serializer.Serializer;
+import org.opensearch.common.cache.stats.CacheStatsHolder;
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.common.cache.stats.NoopCacheStatsHolder;
 import org.opensearch.common.cache.store.builders.ICacheBuilder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.concurrent.ConcurrentHashMap;
@@ -32,12 +36,19 @@ public class MockDiskCache<K, V> implements ICache<K, V> {
     long delay;
 
     private final RemovalListener<ICacheKey<K>, V> removalListener;
+    private final CacheStatsHolder statsHolder; // Only update for number of entries; this is only used to test statsTrackingEnabled logic
+                                                // in TSC
 
-    public MockDiskCache(int maxSize, long delay, RemovalListener<ICacheKey<K>, V> removalListener) {
+    public MockDiskCache(int maxSize, long delay, RemovalListener<ICacheKey<K>, V> removalListener, boolean statsTrackingEnabled) {
         this.maxSize = maxSize;
         this.delay = delay;
         this.removalListener = removalListener;
         this.cache = new ConcurrentHashMap<ICacheKey<K>, V>();
+        if (statsTrackingEnabled) {
+            this.statsHolder = new DefaultCacheStatsHolder(List.of(), "mock_disk_cache");
+        } else {
+            this.statsHolder = NoopCacheStatsHolder.getInstance();
+        }
     }
 
     @Override
@@ -50,6 +61,7 @@ public V get(ICacheKey<K> key) {
     public void put(ICacheKey<K> key, V value) {
         if (this.cache.size() >= maxSize) { // For simplification
             this.removalListener.onRemoval(new RemovalNotification<>(key, value, RemovalReason.EVICTED));
+            this.statsHolder.decrementItems(List.of());
         }
         try {
             Thread.sleep(delay);
@@ -57,6 +69,7 @@ public void put(ICacheKey<K> key, V value) {
             throw new RuntimeException(e);
         }
         this.cache.put(key, value);
+        this.statsHolder.incrementItems(List.of());
     }
 
     @Override
@@ -73,6 +86,7 @@ public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V>
 
     @Override
     public void invalidate(ICacheKey<K> key) {
+        removalListener.onRemoval(new RemovalNotification<>(key, cache.get(key), RemovalReason.INVALIDATED));
         this.cache.remove(key);
     }
 
@@ -96,7 +110,9 @@ public void refresh() {}
 
     @Override
     public ImmutableCacheStatsHolder stats() {
-        return null;
+        // To allow testing of statsTrackingEnabled logic in TSC, return a dummy ImmutableCacheStatsHolder with the
+        // right number of entries, unless statsTrackingEnabled is false
+        return statsHolder.getImmutableCacheStatsHolder(null);
     }
 
     @Override
@@ -114,10 +130,12 @@ public static class MockDiskCacheFactory implements Factory {
         public static final String NAME = "mockDiskCache";
         final long delay;
         final int maxSize;
+        final boolean statsTrackingEnabled;
 
-        public MockDiskCacheFactory(long delay, int maxSize) {
+        public MockDiskCacheFactory(long delay, int maxSize, boolean statsTrackingEnabled) {
             this.delay = delay;
             this.maxSize = maxSize;
+            this.statsTrackingEnabled = statsTrackingEnabled;
         }
 
         @Override
@@ -128,6 +146,7 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
                 .setMaxSize(maxSize)
                 .setDeliberateDelay(delay)
                 .setRemovalListener(config.getRemovalListener())
+                .setStatsTrackingEnabled(config.getStatsTrackingEnabled())
                 .build();
         }
 
@@ -146,7 +165,7 @@ public static class Builder<K, V> extends ICacheBuilder<K, V> {
 
         @Override
         public ICache<K, V> build() {
-            return new MockDiskCache<K, V>(this.maxSize, this.delay, this.getRemovalListener());
+            return new MockDiskCache<K, V>(this.maxSize, this.delay, this.getRemovalListener(), getStatsTrackingEnabled());
         }
 
         public Builder<K, V> setMaxSize(int maxSize) {
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
index 1ecb63414dc68..6d5ee91326338 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.cache.common.tier;
 
+import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
 import org.opensearch.common.cache.ICacheKey;
@@ -16,6 +17,8 @@
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.policy.CachedQueryResult;
 import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.cache.stats.ImmutableCacheStats;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.OpenSearchOnHeapCache;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
@@ -28,11 +31,13 @@
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -44,10 +49,12 @@
 
 import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP;
 import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_NAME;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP;
 import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY;
 
 public class TieredSpilloverCacheTests extends OpenSearchTestCase {
-    // TODO: TSC stats impl is in a future PR. Parts of tests which use stats values are missing for now.
     static final List<String> dimensionNames = List.of("dim1", "dim2", "dim3");
 
     private ClusterSettings clusterSettings;
@@ -89,6 +96,9 @@ public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
         assertEquals(0, removalListener.evictionsMetric.count());
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
 
         // Try to hit cache again with some randomization.
         int numOfItems2 = randomIntBetween(1, onHeapCacheSize / 2 - 1);
@@ -107,6 +117,13 @@ public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception
             }
         }
         assertEquals(0, removalListener.evictionsMetric.count());
+        assertEquals(cacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(numOfItems1 + cacheMiss, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(numOfItems1 + cacheMiss, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
     public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception {
@@ -160,7 +177,7 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception
                 OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                 new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                 MockDiskCache.MockDiskCacheFactory.NAME,
-                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
             )
         );
 
@@ -174,12 +191,25 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception
             LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(getICacheKey(key), tieredCacheLoader);
         }
+
+        int expectedDiskEntries = numOfItems1 - onHeapCacheSize;
         tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add);
         tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add);
-        // Verify on heap cache size.
+        // Verify on heap cache stats.
         assertEquals(onHeapCacheSize, tieredSpilloverCache.getOnHeapCache().count());
-        // Verify disk cache size.
-        assertEquals(numOfItems1 - onHeapCacheSize, tieredSpilloverCache.getDiskCache().count());
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(expectedDiskEntries, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+
+        // Verify disk cache stats.
+        assertEquals(expectedDiskEntries, tieredSpilloverCache.getDiskCache().count());
+        assertEquals(expectedDiskEntries, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(expectedDiskEntries * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
     public void testWithFactoryCreationWithOnHeapCacheNotPresent() {
@@ -222,7 +252,7 @@ public void testWithFactoryCreationWithOnHeapCacheNotPresent() {
                     OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                     new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                     MockDiskCache.MockDiskCacheFactory.NAME,
-                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
                 )
             )
         );
@@ -267,7 +297,7 @@ public void testWithFactoryCreationWithDiskCacheNotPresent() {
                     OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                     new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                     MockDiskCache.MockDiskCacheFactory.NAME,
-                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
                 )
             )
         );
@@ -307,7 +337,7 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
             .setClusterSettings(clusterSettings)
             .build();
 
-        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize);
+        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false);
 
         TieredSpilloverCache<String, String> tieredSpilloverCache = new TieredSpilloverCache.Builder<String, String>()
             .setOnHeapCacheFactory(onHeapCacheFactory)
@@ -327,6 +357,15 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
 
+        long actualDiskCacheSize = tieredSpilloverCache.getDiskCache().count();
+
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(actualDiskCacheSize, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(actualDiskCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+
         tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add);
         tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add);
 
@@ -350,12 +389,18 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
                 assertFalse(loadAwareCacheLoader.isLoaded());
             }
         }
-        for (int iter = 0; iter < randomIntBetween(50, 200); iter++) {
+        int numRandom = randomIntBetween(50, 200);
+        for (int iter = 0; iter < numRandom; iter++) {
             // Hit cache with randomized key which is expected to miss cache always.
             LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader);
             cacheMiss++;
         }
+
+        assertEquals(numOfItems1 + cacheMiss + diskCacheHit, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(cacheMiss + numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(diskCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
     public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
@@ -385,8 +430,13 @@ public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
             tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader);
         }
 
-        int evictions = numOfItems - (totalSize);
+        int evictions = numOfItems - (totalSize); // Evictions from the cache as a whole
         assertEquals(evictions, removalListener.evictionsMetric.count());
+        assertEquals(evictions, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(
+            evictions + getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK),
+            getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)
+        );
     }
 
     public void testGetAndCount() throws Exception {
@@ -442,7 +492,7 @@ public void testGetAndCount() throws Exception {
         assertEquals(numOfItems1, tieredSpilloverCache.count());
     }
 
-    public void testPut() {
+    public void testPut() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
         int keyValueSize = 50;
@@ -465,6 +515,8 @@ public void testPut() {
         ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
         String value = UUID.randomUUID().toString();
         tieredSpilloverCache.put(key, value);
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, tieredSpilloverCache.count());
     }
 
     public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception {
@@ -497,6 +549,9 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception {
             tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), getLoadAwareCacheLoader());
         }
 
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+
         // Again try to put OnHeap cache capacity amount of new items.
         List<ICacheKey<String>> newKeyList = new ArrayList<>();
         for (int i = 0; i < onHeapCacheSize; i++) {
@@ -515,9 +570,11 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception {
         for (int i = 0; i < actualOnHeapCacheKeys.size(); i++) {
             assertTrue(newKeyList.contains(actualOnHeapCacheKeys.get(i)));
         }
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
-    public void testInvalidate() {
+    public void testInvalidate() throws Exception {
         int onHeapCacheSize = 1;
         int diskCacheSize = 10;
         int keyValueSize = 20;
@@ -541,11 +598,12 @@ public void testInvalidate() {
         String value = UUID.randomUUID().toString();
         // First try to invalidate without the key present in cache.
         tieredSpilloverCache.invalidate(key);
-        // assertEquals(0, tieredSpilloverCache.stats().getEvictionsByDimensions(HEAP_DIMS));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
 
         // Now try to invalidate with the key present in onHeap cache.
         tieredSpilloverCache.put(key, value);
         tieredSpilloverCache.invalidate(key);
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         // Evictions metric shouldn't increase for invalidations.
         assertEquals(0, tieredSpilloverCache.count());
 
@@ -555,11 +613,15 @@ public void testInvalidate() {
         tieredSpilloverCache.put(key2, UUID.randomUUID().toString());
 
         assertEquals(2, tieredSpilloverCache.count());
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
 
         // Again invalidate older key, leaving one in heap tier and zero in disk tier
         tieredSpilloverCache.invalidate(key);
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         assertEquals(1, tieredSpilloverCache.count());
-
     }
 
     public void testCacheKeys() throws Exception {
@@ -752,6 +814,9 @@ public String load(ICacheKey<String> key) {
             }
         }
         assertEquals(1, numberOfTimesKeyLoaded); // It should be loaded only once.
+        // We should see only one heap miss, and the rest hits
+        assertEquals(1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(numberOfSameKeys - 1, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
     }
 
     public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exception {
@@ -760,7 +825,7 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
 
         ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory();
-        ICache.Factory diskCacheFactory = new MockDiskCache.MockDiskCacheFactory(500, diskCacheSize);
+        ICache.Factory diskCacheFactory = new MockDiskCache.MockDiskCacheFactory(500, diskCacheSize, false);
         CacheConfig<String, String> cacheConfig = new CacheConfig.Builder<String, String>().setKeyType(String.class)
             .setKeyType(String.class)
             .setWeigher((k, v) -> 150)
@@ -796,7 +861,7 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
 
         // Put first key on tiered cache. Will go into onHeap cache.
         tieredSpilloverCache.computeIfAbsent(keyToBeEvicted, getLoadAwareCacheLoader());
-        // assertEquals(1, tieredSpilloverCache.stats().getEntriesByDimensions(HEAP_DIMS));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         CountDownLatch countDownLatch = new CountDownLatch(1);
         CountDownLatch countDownLatch1 = new CountDownLatch(1);
         // Put second key on tiered cache. Will cause eviction of first key from onHeap cache and should go into
@@ -834,6 +899,10 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
 
         assertEquals(1, tieredSpilloverCache.getOnHeapCache().count());
         assertEquals(1, onDiskCache.count());
+
+        assertEquals(1, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
         assertNotNull(onDiskCache.get(keyToBeEvicted));
     }
 
@@ -961,7 +1030,7 @@ public CachedQueryResult.PolicyValues apply(String s) {
                 OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                 new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                 MockDiskCache.MockDiskCacheFactory.NAME,
-                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
             )
         );
 
@@ -1053,7 +1122,6 @@ public void testGetPutAndInvalidateWithDiskCacheDisabled() throws Exception {
         int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
         int keyValueSize = 50;
         int totalSize = onHeapCacheSize + diskCacheSize;
-
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
         TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
@@ -1122,6 +1190,128 @@ public void testGetPutAndInvalidateWithDiskCacheDisabled() throws Exception {
         assertEquals(0, tieredSpilloverCache.count());
     }
 
+    public void testTiersDoNotTrackStats() throws Exception {
+        int onHeapCacheSize = randomIntBetween(10, 30);
+        int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
+        int keyValueSize = 50;
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            Settings.builder()
+                .put(
+                    OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    onHeapCacheSize * keyValueSize + "b"
+                )
+                .build(),
+            0
+        );
+
+        // do some gets to put entries in both tiers
+        int numMisses = onHeapCacheSize + randomIntBetween(10, 20);
+        for (int iter = 0; iter < numMisses; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+        }
+        assertNotEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.stats().getTotalStats());
+        assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.getOnHeapCache().stats().getTotalStats());
+        ImmutableCacheStats diskStats = tieredSpilloverCache.getDiskCache().stats().getTotalStats();
+        assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), diskStats);
+    }
+
+    public void testTierStatsAddCorrectly() throws Exception {
+        /* We expect the total stats to be:
+         * totalHits = heapHits + diskHits
+         * totalMisses = diskMisses
+         * totalEvictions = diskEvictions
+         * totalSize = heapSize + diskSize
+         * totalEntries = heapEntries + diskEntries
+         */
+
+        int onHeapCacheSize = randomIntBetween(10, 30);
+        int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
+        int keyValueSize = 50;
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            Settings.builder()
+                .put(
+                    OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    onHeapCacheSize * keyValueSize + "b"
+                )
+                .build(),
+            0
+        );
+
+        List<ICacheKey<String>> usedKeys = new ArrayList<>();
+        // Fill the cache, getting some entries + evictions for both tiers
+        int numMisses = onHeapCacheSize + diskCacheSize + randomIntBetween(10, 20);
+        for (int iter = 0; iter < numMisses; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            usedKeys.add(key);
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+        }
+        // Also do some random hits
+        Random rand = Randomness.get();
+        int approxNumHits = 30;
+        for (int i = 0; i < approxNumHits; i++) {
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            ICacheKey<String> key = usedKeys.get(rand.nextInt(usedKeys.size()));
+            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+        }
+
+        ImmutableCacheStats totalStats = tieredSpilloverCache.stats().getTotalStats();
+        ImmutableCacheStats heapStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP);
+        ImmutableCacheStats diskStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK);
+
+        assertEquals(totalStats.getHits(), heapStats.getHits() + diskStats.getHits());
+        assertEquals(totalStats.getMisses(), diskStats.getMisses());
+        assertEquals(totalStats.getEvictions(), diskStats.getEvictions());
+        assertEquals(totalStats.getSizeInBytes(), heapStats.getSizeInBytes() + diskStats.getSizeInBytes());
+        assertEquals(totalStats.getItems(), heapStats.getItems() + diskStats.getItems());
+
+        // Also check the heap stats don't have zero misses or evictions
+        assertNotEquals(0, heapStats.getMisses());
+        assertNotEquals(0, heapStats.getEvictions());
+
+        // Now turn off the disk tier and do more misses and evictions from the heap tier.
+        // These should be added to the totals, as the disk tier is now absent
+        long missesBeforeDisablingDiskCache = totalStats.getMisses();
+        long evictionsBeforeDisablingDiskCache = totalStats.getEvictions();
+        long heapTierEvictionsBeforeDisablingDiskCache = heapStats.getEvictions();
+
+        clusterSettings.applySettings(
+            Settings.builder().put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), false).build()
+        );
+
+        int newMisses = randomIntBetween(10, 30);
+        for (int i = 0; i < newMisses; i++) {
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader);
+        }
+
+        totalStats = tieredSpilloverCache.stats().getTotalStats();
+        heapStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP);
+        assertEquals(missesBeforeDisablingDiskCache + newMisses, totalStats.getMisses());
+        assertEquals(heapTierEvictionsBeforeDisablingDiskCache + newMisses, heapStats.getEvictions());
+        assertEquals(evictionsBeforeDisablingDiskCache + newMisses, totalStats.getEvictions());
+
+        // Turn the disk cache back on in cluster settings for other tests
+        clusterSettings.applySettings(
+            Settings.builder().put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), true).build()
+        );
+
+    }
+
     private List<String> getMockDimensions() {
         List<String> dims = new ArrayList<>();
         for (String dimensionName : dimensionNames) {
@@ -1237,7 +1427,7 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
             )
             .setClusterSettings(clusterSettings)
             .build();
-        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize);
+        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize, false);
 
         TieredSpilloverCache.Builder<String, String> builder = new TieredSpilloverCache.Builder<String, String>().setCacheType(
             CacheType.INDICES_REQUEST_CACHE
@@ -1251,4 +1441,42 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
         }
         return builder.build();
     }
+
+    // Helper functions for extracting tier aggregated stats.
+    private long getHitsForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getHits();
+    }
+
+    private long getMissesForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getMisses();
+    }
+
+    private long getEvictionsForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getEvictions();
+    }
+
+    private long getSizeInBytesForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getSizeInBytes();
+    }
+
+    private long getItemsForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getItems();
+    }
+
+    private ImmutableCacheStats getStatsSnapshotForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        List<String> levelsList = new ArrayList<>(dimensionNames);
+        levelsList.add(TIER_DIMENSION_NAME);
+        String[] levels = levelsList.toArray(new String[0]);
+        ImmutableCacheStatsHolder cacheStats = tsc.stats(levels);
+        // Since we always use the same list of dimensions from getMockDimensions() in keys for these tests, we can get all the stats values
+        // for a given tier with a single node in MDCS
+        List<String> mockDimensions = getMockDimensions();
+        mockDimensions.add(tierValue);
+        ImmutableCacheStats snapshot = cacheStats.getStatsForDimensionValues(mockDimensions);
+        if (snapshot == null) {
+            return new ImmutableCacheStats(0, 0, 0, 0, 0); // This can happen if no cache actions have happened for this set of
+            // dimensions yet
+        }
+        return snapshot;
+    }
 }
diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
index 93c54a48d59da..9a4dce1067b61 100644
--- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
+++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
@@ -27,6 +27,7 @@
 import org.opensearch.common.cache.stats.CacheStatsHolder;
 import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.common.cache.stats.NoopCacheStatsHolder;
 import org.opensearch.common.cache.store.builders.ICacheBuilder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.collect.Tuple;
@@ -163,8 +164,13 @@ private EhcacheDiskCache(Builder<K, V> builder) {
         this.ehCacheEventListener = new EhCacheEventListener(builder.getRemovalListener(), builder.getWeigher());
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
         List<String> dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null");
-        // If this cache is being used, FeatureFlags.PLUGGABLE_CACHE is already on, so we can always use the DefaultCacheStatsHolder.
-        this.cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME);
+        if (builder.getStatsTrackingEnabled()) {
+            // If this cache is being used, FeatureFlags.PLUGGABLE_CACHE is already on, so we can always use the DefaultCacheStatsHolder
+            // unless statsTrackingEnabled is explicitly set to false in CacheConfig.
+            this.cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME);
+        } else {
+            this.cacheStatsHolder = NoopCacheStatsHolder.getInstance();
+        }
     }
 
     @SuppressWarnings({ "rawtypes" })
@@ -414,6 +420,7 @@ public Iterable<ICacheKey<K>> keys() {
 
     /**
      * Gives the current count of keys in disk cache.
+     * If enableStatsTracking is set to false in the builder, always returns 0.
      * @return current count of keys
      */
     @Override
diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
index f2bfe1209a4c7..29551befd3e9f 100644
--- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
+++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
@@ -850,6 +850,38 @@ public void testInvalidateWithDropDimensions() throws Exception {
         }
     }
 
+    public void testStatsTrackingDisabled() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setIsEventListenerModeSync(true)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .setWeigher(weigher)
+                .setStatsTrackingEnabled(false)
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            for (int i = 0; i < randomKeys; i++) {
+                ICacheKey<String> iCacheKey = getICacheKey(UUID.randomUUID().toString());
+                ehcacheTest.put(iCacheKey, UUID.randomUUID().toString());
+                assertEquals(0, ehcacheTest.count()); // Expect count of 0 if NoopCacheStatsHolder is used
+                assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), ehcacheTest.stats().getTotalStats());
+            }
+            ehcacheTest.close();
+        }
+    }
+
     private List<String> getRandomDimensions(List<String> dimensionNames) {
         Random rand = Randomness.get();
         int bound = 3;
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java
index 5574e345b6d3d..ea92c8e81b8f0 100644
--- a/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java
+++ b/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java
@@ -32,7 +32,7 @@ public class DefaultCacheStatsHolder implements CacheStatsHolder {
 
     // The list of permitted dimensions. Should be ordered from "outermost" to "innermost", as you would like to
     // aggregate them in an API response.
-    private final List<String> dimensionNames;
+    protected final List<String> dimensionNames;
     // A tree structure based on dimension values, which stores stats values in its leaf nodes.
     // Non-leaf nodes have stats matching the sum of their children.
     // We use a tree structure, rather than a map with concatenated keys, to save on memory usage. If there are many leaf
@@ -115,7 +115,7 @@ public long count() {
         return statsRoot.getEntries();
     }
 
-    private void internalIncrement(List<String> dimensionValues, Consumer<Node> adder, boolean createNodesIfAbsent) {
+    protected void internalIncrement(List<String> dimensionValues, Consumer<Node> adder, boolean createNodesIfAbsent) {
         assert dimensionValues.size() == dimensionNames.size();
         // First try to increment without creating nodes
         boolean didIncrement = internalIncrementHelper(dimensionValues, statsRoot, 0, adder, false);
@@ -213,7 +213,10 @@ Node getStatsRoot() {
         return statsRoot;
     }
 
-    static class Node {
+    /**
+     * Nodes that make up the tree in the stats holder.
+     */
+    protected static class Node {
         private final String dimensionValue;
         // Map from dimensionValue to the DimensionNode for that dimension value.
         final Map<String, Node> children;
@@ -245,23 +248,23 @@ protected Map<String, Node> getChildren() {
 
         // Functions for modifying internal CacheStatsCounter without callers having to be aware of CacheStatsCounter
 
-        void incrementHits() {
+        public void incrementHits() {
             this.stats.incrementHits();
         }
 
-        void incrementMisses() {
+        public void incrementMisses() {
             this.stats.incrementMisses();
         }
 
-        void incrementEvictions() {
+        public void incrementEvictions() {
             this.stats.incrementEvictions();
         }
 
-        void incrementSizeInBytes(long amountBytes) {
+        public void incrementSizeInBytes(long amountBytes) {
             this.stats.incrementSizeInBytes(amountBytes);
         }
 
-        void decrementSizeInBytes(long amountBytes) {
+        public void decrementSizeInBytes(long amountBytes) {
             this.stats.decrementSizeInBytes(amountBytes);
         }
 
@@ -296,5 +299,16 @@ Node getChild(String dimensionValue) {
         Node createChild(String dimensionValue, boolean createMapInChild) {
             return children.computeIfAbsent(dimensionValue, (key) -> new Node(dimensionValue, createMapInChild));
         }
+
+        /**
+         * Return whether this is a leaf node which is at the lowest level of the tree.
+         * Does not return true if this is a node at a higher level whose children are still being constructed.
+         * @return if this is a leaf node at the lowest level
+         */
+        public boolean isAtLowestLevel() {
+            // Compare by value to the empty children map, to ensure we don't get false positives for nodes
+            // which are in the process of having children added
+            return children == EMPTY_CHILDREN_MAP;
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java
index f4cf9f3a8fa61..569653bec2a3d 100644
--- a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java
+++ b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java
@@ -53,6 +53,7 @@ public class OpenSearchOnHeapCache<K, V> implements ICache<K, V>, RemovalListene
     private final RemovalListener<ICacheKey<K>, V> removalListener;
     private final List<String> dimensionNames;
     private final ToLongBiFunction<ICacheKey<K>, V> weigher;
+    private final boolean statsTrackingEnabled;
 
     public OpenSearchOnHeapCache(Builder<K, V> builder) {
         CacheBuilder<ICacheKey<K>, V> cacheBuilder = CacheBuilder.<ICacheKey<K>, V>builder()
@@ -64,12 +65,11 @@ public OpenSearchOnHeapCache(Builder<K, V> builder) {
         }
         cache = cacheBuilder.build();
         this.dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null");
-        // Use noop stats when pluggable caching is off
-        boolean useNoopStats = !FeatureFlags.PLUGGABLE_CACHE_SETTING.get(builder.getSettings());
-        if (useNoopStats) {
-            this.cacheStatsHolder = NoopCacheStatsHolder.getInstance();
-        } else {
+        this.statsTrackingEnabled = builder.getStatsTrackingEnabled();
+        if (statsTrackingEnabled) {
             this.cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, OpenSearchOnHeapCacheFactory.NAME);
+        } else {
+            this.cacheStatsHolder = NoopCacheStatsHolder.getInstance();
         }
         this.removalListener = builder.getRemovalListener();
         this.weigher = builder.getWeigher();
@@ -171,8 +171,9 @@ public static class OpenSearchOnHeapCacheFactory implements Factory {
         public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType, Map<String, Factory> cacheFactories) {
             Map<String, Setting<?>> settingList = OpenSearchOnHeapCacheSettings.getSettingListForCacheType(cacheType);
             Settings settings = config.getSettings();
+            boolean statsTrackingEnabled = statsTrackingEnabled(config.getSettings(), config.getStatsTrackingEnabled());
             ICacheBuilder<K, V> builder = new Builder<K, V>().setDimensionNames(config.getDimensionNames())
-                .setSettings(config.getSettings())
+                .setStatsTrackingEnabled(statsTrackingEnabled)
                 .setMaximumWeightInBytes(((ByteSizeValue) settingList.get(MAXIMUM_SIZE_IN_BYTES_KEY).get(settings)).getBytes())
                 .setExpireAfterAccess(((TimeValue) settingList.get(EXPIRE_AFTER_ACCESS_KEY).get(settings)))
                 .setWeigher(config.getWeigher())
@@ -193,6 +194,11 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
         public String getCacheName() {
             return NAME;
         }
+
+        private boolean statsTrackingEnabled(Settings settings, boolean statsTrackingEnabledConfig) {
+            // Don't track stats when pluggable caching is off, or when explicitly set to false in the CacheConfig
+            return FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings) && statsTrackingEnabledConfig;
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java b/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java
index ac90fcc85ffef..a308d1db88258 100644
--- a/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java
+++ b/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java
@@ -37,6 +37,8 @@ public abstract class ICacheBuilder<K, V> {
 
     private RemovalListener<ICacheKey<K>, V> removalListener;
 
+    private boolean statsTrackingEnabled = true;
+
     public ICacheBuilder() {}
 
     public ICacheBuilder<K, V> setMaximumWeightInBytes(long sizeInBytes) {
@@ -64,6 +66,11 @@ public ICacheBuilder<K, V> setRemovalListener(RemovalListener<ICacheKey<K>, V> r
         return this;
     }
 
+    public ICacheBuilder<K, V> setStatsTrackingEnabled(boolean statsTrackingEnabled) {
+        this.statsTrackingEnabled = statsTrackingEnabled;
+        return this;
+    }
+
     public long getMaxWeightInBytes() {
         return maxWeightInBytes;
     }
@@ -84,5 +91,9 @@ public Settings getSettings() {
         return settings;
     }
 
+    public boolean getStatsTrackingEnabled() {
+        return statsTrackingEnabled;
+    }
+
     public abstract ICache<K, V> build();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java b/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java
index 15cbdbd021d71..0c54ac57a9b18 100644
--- a/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java
+++ b/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java
@@ -68,6 +68,8 @@ public class CacheConfig<K, V> {
 
     private final ClusterSettings clusterSettings;
 
+    private final boolean statsTrackingEnabled;
+
     private CacheConfig(Builder<K, V> builder) {
         this.keyType = builder.keyType;
         this.valueType = builder.valueType;
@@ -81,6 +83,7 @@ private CacheConfig(Builder<K, V> builder) {
         this.maxSizeInBytes = builder.maxSizeInBytes;
         this.expireAfterAccess = builder.expireAfterAccess;
         this.clusterSettings = builder.clusterSettings;
+        this.statsTrackingEnabled = builder.statsTrackingEnabled;
     }
 
     public Class<K> getKeyType() {
@@ -131,6 +134,10 @@ public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
 
+    public boolean getStatsTrackingEnabled() {
+        return statsTrackingEnabled;
+    }
+
     /**
      * Builder class to build Cache config related parameters.
      * @param <K> Type of key.
@@ -155,6 +162,7 @@ public static class Builder<K, V> {
 
         private TimeValue expireAfterAccess;
         private ClusterSettings clusterSettings;
+        private boolean statsTrackingEnabled = true;
 
         public Builder() {}
 
@@ -218,6 +226,11 @@ public Builder<K, V> setClusterSettings(ClusterSettings clusterSettings) {
             return this;
         }
 
+        public Builder<K, V> setStatsTrackingEnabled(boolean statsTrackingEnabled) {
+            this.statsTrackingEnabled = statsTrackingEnabled;
+            return this;
+        }
+
         public CacheConfig<K, V> build() {
             return new CacheConfig<>(this);
         }
diff --git a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java
index f5885d03f1850..f227db6fee2d1 100644
--- a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java
+++ b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java
@@ -39,7 +39,7 @@ public void testStats() throws Exception {
         MockRemovalListener<String, String> listener = new MockRemovalListener<>();
         int maxKeys = between(10, 50);
         int numEvicted = between(10, 20);
-        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, true);
+        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, true, true);
 
         // When the pluggable caches setting is on, we should get stats as expected from cache.stats().
 
@@ -82,21 +82,26 @@ public void testStats() throws Exception {
     }
 
     public void testStatsWithoutPluggableCaches() throws Exception {
-        // When the pluggable caches setting is off, we should get all-zero stats from cache.stats(), but count() should still work.
+        // When the pluggable caches setting is off, or when we manually set statsTrackingEnabled = false in the config,
+        // we should get all-zero stats from cache.stats(), but count() should still work.
         MockRemovalListener<String, String> listener = new MockRemovalListener<>();
         int maxKeys = between(10, 50);
         int numEvicted = between(10, 20);
-        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, false);
 
-        List<ICacheKey<String>> keysAdded = new ArrayList<>();
-        int numAdded = maxKeys + numEvicted;
-        for (int i = 0; i < numAdded; i++) {
-            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
-            keysAdded.add(key);
-            cache.computeIfAbsent(key, getLoadAwareCacheLoader());
+        OpenSearchOnHeapCache<String, String> pluggableCachesOffCache = getCache(maxKeys, listener, false, true);
+        OpenSearchOnHeapCache<String, String> manuallySetNoopStatsCache = getCache(maxKeys, listener, true, false);
+        List<OpenSearchOnHeapCache<String, String>> caches = List.of(pluggableCachesOffCache, manuallySetNoopStatsCache);
 
-            assertEquals(Math.min(maxKeys, i + 1), cache.count());
-            assertZeroStats(cache.stats());
+        for (OpenSearchOnHeapCache<String, String> cache : caches) {
+            int numAdded = maxKeys + numEvicted;
+            for (int i = 0; i < numAdded; i++) {
+                ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+                cache.computeIfAbsent(key, getLoadAwareCacheLoader());
+
+                assertEquals(Math.min(maxKeys, i + 1), cache.count());
+                ImmutableCacheStatsHolder stats = cache.stats();
+                assertZeroStats(cache.stats());
+            }
         }
     }
 
@@ -107,7 +112,8 @@ private void assertZeroStats(ImmutableCacheStatsHolder stats) {
     private OpenSearchOnHeapCache<String, String> getCache(
         int maxSizeKeys,
         MockRemovalListener<String, String> listener,
-        boolean pluggableCachesSetting
+        boolean pluggableCachesSetting,
+        boolean statsTrackingEnabled
     ) {
         ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory();
         Settings settings = Settings.builder()
@@ -127,6 +133,7 @@ private OpenSearchOnHeapCache<String, String> getCache(
             .setSettings(settings)
             .setDimensionNames(dimensionNames)
             .setMaxSizeInBytes(maxSizeKeys * keyValueSize)
+            .setStatsTrackingEnabled(statsTrackingEnabled)
             .build();
         return (OpenSearchOnHeapCache<String, String>) onHeapCacheFactory.create(cacheConfig, CacheType.INDICES_REQUEST_CACHE, null);
     }
@@ -134,7 +141,7 @@ private OpenSearchOnHeapCache<String, String> getCache(
     public void testInvalidateWithDropDimensions() throws Exception {
         MockRemovalListener<String, String> listener = new MockRemovalListener<>();
         int maxKeys = 50;
-        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, true);
+        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, true, true);
 
         List<ICacheKey<String>> keysAdded = new ArrayList<>();
 

From ed33488aa426bd618685729fc638adad763f6ff7 Mon Sep 17 00:00:00 2001
From: bowenlan-amzn <bowenlan23@gmail.com>
Date: Thu, 2 May 2024 19:35:32 -0700
Subject: [PATCH 11/52] Support multi ranges traversal when doing date
 histogram rewrite optimization (#13317)

---
 CHANGELOG.md                                  |   1 +
 .../test/search.aggregation/10_histogram.yml  |  56 ++
 .../test/search.aggregation/230_composite.yml |  58 ++
 .../330_auto_date_histogram.yml               |  26 +
 .../org/opensearch/search/SearchService.java  |   2 +-
 .../bucket/FastFilterRewriteHelper.java       | 502 +++++++++++++-----
 .../bucket/composite/CompositeAggregator.java |  19 +-
 .../AutoDateHistogramAggregator.java          |  14 +-
 .../histogram/DateHistogramAggregator.java    |   9 +-
 .../DateHistogramAggregatorTests.java         | 274 ++++++++++
 .../aggregations/AggregatorTestCase.java      |   4 +-
 11 files changed, 821 insertions(+), 144 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cdaa1904e2522..3f2b92c76689f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))
 - [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276))
 - Add support for deep copying SearchRequest ([#12295](https://github.com/opensearch-project/OpenSearch/pull/12295))
+- Support multi ranges traversal when doing date histogram rewrite optimization. ([#13317](https://github.com/opensearch-project/OpenSearch/pull/13317))
 
 ### Dependencies
 - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index e7da9a0bc454c..fa71137912a91 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -644,3 +644,59 @@ setup:
   - match: { aggregations.histo.buckets.0.doc_count: 1 }
   - match: { aggregations.histo.buckets.20.key: 20 }
   - match: { aggregations.histo.buckets.20.doc_count: 1 }
+
+---
+"date_histogram profiler shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason:  debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+  - do:
+      indices.create:
+        index: test_2
+        body:
+          settings:
+            number_of_replicas: 0
+            number_of_shards: 1
+          mappings:
+            properties:
+              date:
+                type: date
+
+  - do:
+      bulk:
+        index: test_2
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"date": "2016-01-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-01-02"}'
+          - '{"index": {}}'
+          - '{"date": "2016-02-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-03-01"}'
+
+  - do:
+      search:
+        index: test_2
+        body:
+          size: 0
+          profile: true
+          aggs:
+            histo:
+              date_histogram:
+                field: date
+                calendar_interval: month
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 3 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "2016-01-01T00:00:00.000Z" }
+  - match: { aggregations.histo.buckets.0.doc_count: 2 }
+  - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator }
+  - match: { profile.shards.0.aggregations.0.description: histo }
+  - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 }
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
index 2808be8cd7045..3a0099dae3b33 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
@@ -1069,3 +1069,61 @@ setup:
   - match: { aggregations.test.buckets.1.doc_count: 2 }
   - match: { aggregations.test.buckets.2.key.kw: null }
   - match: { aggregations.test.buckets.2.doc_count: 2 }
+
+---
+"composite aggregation date_histogram profile shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason:  debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+  - do:
+      indices.create:
+        index: test_2
+        body:
+          settings:
+            number_of_replicas: 0
+            number_of_shards: 1
+          mappings:
+            properties:
+              date:
+                type: date
+  - do:
+      bulk:
+        index: test_2
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"date": "2016-01-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-01-02"}'
+          - '{"index": {}}'
+          - '{"date": "2016-02-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-03-01"}'
+  - do:
+      search:
+        index: test_2
+        body:
+          size: 0
+          profile: true
+          aggregations:
+            test:
+              composite:
+                sources: [
+                  {
+                    "date": {
+                      "date_histogram": {
+                        "field": "date",
+                        "calendar_interval": "1d",
+                        "format": "strict_date"
+                      }
+                    }
+                  }
+                ]
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.test.buckets: 4 }
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
index 6b5e06a549be3..1356eac41ae79 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
@@ -99,3 +99,29 @@ setup:
   - length: { aggregations.histo.buckets: 2 }
   - match: { profile.shards.0.aggregations.0.type: AutoDateHistogramAggregator.FromSingle }
   - match: { profile.shards.0.aggregations.0.debug.surviving_buckets: 4 }
+
+---
+"auto_date_histogram profile shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+  - do:
+      search:
+        body:
+          profile: true
+          size: 0
+          aggs:
+            histo:
+              auto_date_histogram:
+                field: date
+                buckets: 2
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 2 }
+  - match: { profile.shards.0.aggregations.0.type: AutoDateHistogramAggregator.FromSingle }
+  - match: { profile.shards.0.aggregations.0.debug.surviving_buckets: 4 }
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index 6b3620e65a271..744d3a19f1593 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -275,7 +275,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
     // value 0 means rewrite filters optimization in aggregations will be disabled
     public static final Setting<Integer> MAX_AGGREGATION_REWRITE_FILTERS = Setting.intSetting(
         "search.max_aggregation_rewrite_filters",
-        72,
+        3000,
         0,
         Property.Dynamic,
         Property.NodeScope
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
index dde748bf0dc07..c8ce39a52f869 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
@@ -10,12 +10,13 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.PointValues;
+import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -23,24 +24,30 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.NumericUtils;
+import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.Rounding;
 import org.opensearch.common.lucene.search.function.FunctionScoreQuery;
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.query.DateRangeIncludingNowQuery;
+import org.opensearch.search.aggregations.bucket.composite.CompositeAggregator;
 import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceConfig;
 import org.opensearch.search.aggregations.bucket.composite.RoundingValuesSource;
 import org.opensearch.search.aggregations.bucket.histogram.LongBounds;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalLong;
 import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -128,9 +135,10 @@ private static long[] getSegmentBounds(final LeafReaderContext context, final St
     }
 
     /**
-     * This method also acts as a pre-condition check for the optimization
+     * Gets the min and max bounds of the field for the shard search
+     * Depending on the query part, the bounds are computed differently
      *
-     * @return null if the processed query not as expected
+     * @return null if the processed query not supported by the optimization
      */
     public static long[] getDateHistoAggBounds(final SearchContext context, final String fieldName) throws IOException {
         final Query cq = unwrapIntoConcreteQuery(context.query());
@@ -167,67 +175,6 @@ private static long[] getBoundsWithRangeQuery(PointRangeQuery prq, String fieldN
         return null;
     }
 
-    /**
-     * Creates the date range filters for aggregations using the interval, min/max
-     * bounds and prepared rounding
-     */
-    private static Weight[] createFilterForAggregations(
-        final SearchContext context,
-        final DateFieldMapper.DateFieldType fieldType,
-        final long interval,
-        final Rounding.Prepared preparedRounding,
-        long low,
-        final long high
-    ) throws IOException {
-        // Calculate the number of buckets using range and interval
-        long roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
-        long prevRounded = roundedLow;
-        int bucketCount = 0;
-        while (roundedLow <= fieldType.convertNanosToMillis(high)) {
-            bucketCount++;
-            int maxNumFilterBuckets = context.maxAggRewriteFilters();
-            if (bucketCount > maxNumFilterBuckets) {
-                logger.debug("Max number of filters reached [{}], skip the fast filter optimization", maxNumFilterBuckets);
-                return null;
-            }
-            // Below rounding is needed as the interval could return in
-            // non-rounded values for something like calendar month
-            roundedLow = preparedRounding.round(roundedLow + interval);
-            if (prevRounded == roundedLow) break; // prevents getting into an infinite loop
-            prevRounded = roundedLow;
-        }
-
-        Weight[] filters = null;
-        if (bucketCount > 0) {
-            filters = new Weight[bucketCount];
-            roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
-
-            int i = 0;
-            while (i < bucketCount) {
-                // Calculate the lower bucket bound
-                final byte[] lower = new byte[8];
-                NumericUtils.longToSortableBytes(i == 0 ? low : fieldType.convertRoundedMillisToNanos(roundedLow), lower, 0);
-
-                // Calculate the upper bucket bound
-                roundedLow = preparedRounding.round(roundedLow + interval);
-                final byte[] upper = new byte[8];
-                NumericUtils.longToSortableBytes(i + 1 == bucketCount ? high :
-                // Subtract -1 if the minimum is roundedLow as roundedLow itself
-                // is included in the next bucket
-                    fieldType.convertRoundedMillisToNanos(roundedLow) - 1, upper, 0);
-
-                filters[i++] = context.searcher().createWeight(new PointRangeQuery(fieldType.name(), lower, upper, 1) {
-                    @Override
-                    protected String toString(int dimension, byte[] value) {
-                        return Long.toString(LongPoint.decodeDimension(value, 0));
-                    }
-                }, ScoreMode.COMPLETE_NO_SCORES, 1);
-            }
-        }
-
-        return filters;
-    }
-
     /**
      * Context object for fast filter optimization
      * <p>
@@ -235,12 +182,24 @@ protected String toString(int dimension, byte[] value) {
      */
     public static class FastFilterContext {
         private boolean rewriteable = false;
-        private Weight[] filters = null;
-        private boolean filtersBuiltAtShardLevel = false;
+        private boolean rangesBuiltAtShardLevel = false;
 
         private AggregationType aggregationType;
         private final SearchContext context;
 
+        private String fieldName;
+        private long[][] ranges;
+
+        // debug info related fields
+        public int leaf;
+        public int inner;
+        public int segments;
+        public int optimizedSegments;
+
+        public void setFieldName(String fieldName) {
+            this.fieldName = fieldName;
+        }
+
         public FastFilterContext(SearchContext context) {
             this.context = context;
         }
@@ -262,24 +221,26 @@ public boolean isRewriteable(final Object parent, final int subAggLength) {
             return rewriteable;
         }
 
-        public void buildFastFilter() throws IOException {
-            assert filters == null : "Filters should only be built once, but they are already built";
-            this.filters = this.aggregationType.buildFastFilter(context);
-            if (filters != null) {
-                logger.debug("Fast filter built for shard {}", context.indexShard().shardId());
-                filtersBuiltAtShardLevel = true;
+        public void buildRanges() throws IOException {
+            assert ranges == null : "Ranges should only be built once at shard level, but they are already built";
+            this.ranges = this.aggregationType.buildRanges(context);
+            if (ranges != null) {
+                logger.debug("Ranges built for shard {}", context.indexShard().shardId());
+                rangesBuiltAtShardLevel = true;
             }
         }
 
-        /**
-         * Built filters for a segment
-         */
-        public Weight[] buildFastFilter(LeafReaderContext leaf) throws IOException {
-            Weight[] filters = this.aggregationType.buildFastFilter(leaf, context);
-            if (filters != null) {
-                logger.debug("Fast filter built for shard {} segment {}", context.indexShard().shardId(), leaf.ord);
+        public long[][] buildRanges(LeafReaderContext leaf) throws IOException {
+            long[][] ranges = this.aggregationType.buildRanges(leaf, context);
+            if (ranges != null) {
+                logger.debug("Ranges built for shard {} segment {}", context.indexShard().shardId(), leaf.ord);
             }
-            return filters;
+            return ranges;
+        }
+
+        private void consumeDebugInfo(DebugInfo debug) {
+            leaf += debug.leaf;
+            inner += debug.inner;
         }
     }
 
@@ -287,16 +248,11 @@ public Weight[] buildFastFilter(LeafReaderContext leaf) throws IOException {
      * Different types have different pre-conditions, filter building logic, etc.
      */
     interface AggregationType {
-
         boolean isRewriteable(Object parent, int subAggLength);
 
-        Weight[] buildFastFilter(SearchContext ctx) throws IOException;
-
-        Weight[] buildFastFilter(LeafReaderContext leaf, SearchContext ctx) throws IOException;
+        long[][] buildRanges(SearchContext ctx) throws IOException;
 
-        default int getSize() {
-            return Integer.MAX_VALUE;
-        }
+        long[][] buildRanges(LeafReaderContext leaf, SearchContext ctx) throws IOException;
     }
 
     /**
@@ -330,20 +286,13 @@ public boolean isRewriteable(Object parent, int subAggLength) {
         }
 
         @Override
-        public Weight[] buildFastFilter(SearchContext context) throws IOException {
+        public long[][] buildRanges(SearchContext context) throws IOException {
             long[] bounds = getDateHistoAggBounds(context, fieldType.name());
             logger.debug("Bounds are {} for shard {}", bounds, context.indexShard().shardId());
-            return buildFastFilter(context, bounds);
+            return buildRanges(context, bounds);
         }
 
-        @Override
-        public Weight[] buildFastFilter(LeafReaderContext leaf, SearchContext context) throws IOException {
-            long[] bounds = getSegmentBounds(leaf, fieldType.name());
-            logger.debug("Bounds are {} for shard {} segment {}", bounds, context.indexShard().shardId(), leaf.ord);
-            return buildFastFilter(context, bounds);
-        }
-
-        private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IOException {
+        private long[][] buildRanges(SearchContext context, long[] bounds) throws IOException {
             bounds = processHardBounds(bounds);
             if (bounds == null) {
                 return null;
@@ -360,7 +309,7 @@ private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IO
             // process the after key of composite agg
             processAfterKey(bounds, interval);
 
-            return FastFilterRewriteHelper.createFilterForAggregations(
+            return FastFilterRewriteHelper.createRangesFromAgg(
                 context,
                 (DateFieldMapper.DateFieldType) fieldType,
                 interval,
@@ -370,6 +319,13 @@ private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IO
             );
         }
 
+        @Override
+        public long[][] buildRanges(LeafReaderContext leaf, SearchContext context) throws IOException {
+            long[] bounds = getSegmentBounds(leaf, fieldType.name());
+            logger.debug("Bounds are {} for shard {} segment {}", bounds, context.indexShard().shardId(), leaf.ord);
+            return buildRanges(context, bounds);
+        }
+
         protected abstract Rounding getRounding(final long low, final long high);
 
         protected abstract Rounding.Prepared getRoundingPrepared();
@@ -413,7 +369,7 @@ public static long getBucketOrd(long bucketOrd) {
     }
 
     /**
-     * Try to get the bucket doc counts from the fast filters for the aggregation
+     * Try to get the bucket doc counts for the date histogram aggregation
      * <p>
      * Usage: invoked at segment level — in getLeafCollector of aggregator
      *
@@ -424,11 +380,18 @@ public static boolean tryFastFilterAggregation(
         FastFilterContext fastFilterContext,
         final BiConsumer<Long, Integer> incrementDocCount
     ) throws IOException {
-        if (fastFilterContext == null) return false;
+        fastFilterContext.segments++;
         if (!fastFilterContext.rewriteable) {
             return false;
         }
 
+        if (ctx.reader().hasDeletions()) return false;
+
+        PointValues values = ctx.reader().getPointValues(fastFilterContext.fieldName);
+        if (values == null) return false;
+        // only proceed if every document corresponds to exactly one point
+        if (values.getDocCount() != values.size()) return false;
+
         NumericDocValues docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME);
         if (docCountValues.nextDoc() != NO_MORE_DOCS) {
             logger.debug(
@@ -439,61 +402,330 @@ public static boolean tryFastFilterAggregation(
             return false;
         }
 
-        // if no filters built at shard level (see getDateHistoAggBounds method for possible reasons)
-        // check if the query is functionally match-all at segment level
-        if (!fastFilterContext.filtersBuiltAtShardLevel && !segmentMatchAll(fastFilterContext.context, ctx)) {
+        // even if no ranges built at shard level, we can still perform the optimization
+        // when functionally match-all at segment level
+        if (!fastFilterContext.rangesBuiltAtShardLevel && !segmentMatchAll(fastFilterContext.context, ctx)) {
             return false;
         }
-        Weight[] filters = fastFilterContext.filters;
-        if (filters == null) {
+        long[][] ranges = fastFilterContext.ranges;
+        if (ranges == null) {
             logger.debug(
                 "Shard {} segment {} functionally match all documents. Build the fast filter",
                 fastFilterContext.context.indexShard().shardId(),
                 ctx.ord
             );
-            filters = fastFilterContext.buildFastFilter(ctx);
-            if (filters == null) {
+            ranges = fastFilterContext.buildRanges(ctx);
+            if (ranges == null) {
                 return false;
             }
         }
 
-        final int[] counts = new int[filters.length];
-        int i;
-        for (i = 0; i < filters.length; i++) {
-            counts[i] = filters[i].count(ctx);
-            if (counts[i] == -1) {
-                // Cannot use the optimization if any of the counts
-                // is -1 indicating the segment might have deleted documents
-                return false;
+        final AggregationType aggregationType = fastFilterContext.aggregationType;
+        assert aggregationType instanceof AbstractDateHistogramAggregationType;
+        final DateFieldMapper.DateFieldType fieldType = ((AbstractDateHistogramAggregationType) aggregationType).getFieldType();
+        int size = Integer.MAX_VALUE;
+        if (aggregationType instanceof CompositeAggregator.CompositeAggregationType) {
+            size = ((CompositeAggregator.CompositeAggregationType) aggregationType).getSize();
+        }
+        DebugInfo debugInfo = multiRangesTraverse(values.getPointTree(), ranges, incrementDocCount, fieldType, size);
+        fastFilterContext.consumeDebugInfo(debugInfo);
+
+        fastFilterContext.optimizedSegments++;
+        logger.debug("Fast filter optimization applied to shard {} segment {}", fastFilterContext.context.indexShard().shardId(), ctx.ord);
+        logger.debug("crossed leaf nodes: {}, inner nodes: {}", fastFilterContext.leaf, fastFilterContext.inner);
+        return true;
+    }
+
+    private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leafCtx) throws IOException {
+        Weight weight = ctx.searcher().createWeight(ctx.query(), ScoreMode.COMPLETE_NO_SCORES, 1f);
+        return weight != null && weight.count(leafCtx) == leafCtx.reader().numDocs();
+    }
+
+    /**
+     * Creates the date ranges from date histo aggregations using its interval,
+     * and min/max boundaries
+     */
+    private static long[][] createRangesFromAgg(
+        final SearchContext context,
+        final DateFieldMapper.DateFieldType fieldType,
+        final long interval,
+        final Rounding.Prepared preparedRounding,
+        long low,
+        final long high
+    ) {
+        // Calculate the number of buckets using range and interval
+        long roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
+        long prevRounded = roundedLow;
+        int bucketCount = 0;
+        while (roundedLow <= fieldType.convertNanosToMillis(high)) {
+            bucketCount++;
+            int maxNumFilterBuckets = context.maxAggRewriteFilters();
+            if (bucketCount > maxNumFilterBuckets) {
+                logger.debug("Max number of filters reached [{}], skip the fast filter optimization", maxNumFilterBuckets);
+                return null;
+            }
+            // Below rounding is needed as the interval could return in
+            // non-rounded values for something like calendar month
+            roundedLow = preparedRounding.round(roundedLow + interval);
+            if (prevRounded == roundedLow) break; // prevents getting into an infinite loop
+            prevRounded = roundedLow;
+        }
+
+        long[][] ranges = new long[bucketCount][2];
+        if (bucketCount > 0) {
+            roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
+
+            int i = 0;
+            while (i < bucketCount) {
+                // Calculate the lower bucket bound
+                long lower = i == 0 ? low : fieldType.convertRoundedMillisToNanos(roundedLow);
+                roundedLow = preparedRounding.round(roundedLow + interval);
+
+                // Subtract -1 if the minimum is roundedLow as roundedLow itself
+                // is included in the next bucket
+                long upper = i + 1 == bucketCount ? high : fieldType.convertRoundedMillisToNanos(roundedLow) - 1;
+
+                ranges[i][0] = lower;
+                ranges[i][1] = upper;
+                i++;
+            }
+        }
+
+        return ranges;
+    }
+
+    /**
+     * @param maxNumNonZeroRanges the number of non-zero ranges to collect
+     */
+    private static DebugInfo multiRangesTraverse(
+        final PointValues.PointTree tree,
+        final long[][] ranges,
+        final BiConsumer<Long, Integer> incrementDocCount,
+        final DateFieldMapper.DateFieldType fieldType,
+        final int maxNumNonZeroRanges
+    ) throws IOException {
+        // ranges are connected and in ascending order
+        Iterator<long[]> rangeIter = Arrays.stream(ranges).iterator();
+        long[] activeRange = rangeIter.next();
+
+        // make sure the first range at least crosses the min value of the tree
+        DebugInfo debugInfo = new DebugInfo();
+        if (activeRange[0] > NumericUtils.sortableBytesToLong(tree.getMaxPackedValue(), 0)) {
+            logger.debug("No ranges match the query, skip the fast filter optimization");
+            return debugInfo;
+        }
+        while (activeRange[1] < NumericUtils.sortableBytesToLong(tree.getMinPackedValue(), 0)) {
+            if (!rangeIter.hasNext()) {
+                logger.debug("No ranges match the query, skip the fast filter optimization");
+                return debugInfo;
             }
+            activeRange = rangeIter.next();
         }
 
-        int s = 0;
-        int size = fastFilterContext.aggregationType.getSize();
-        for (i = 0; i < filters.length; i++) {
-            if (counts[i] > 0) {
-                long bucketKey = i; // the index of filters is the key for filters aggregation
-                if (fastFilterContext.aggregationType instanceof AbstractDateHistogramAggregationType) {
-                    final DateFieldMapper.DateFieldType fieldType =
-                        ((AbstractDateHistogramAggregationType) fastFilterContext.aggregationType).getFieldType();
-                    bucketKey = fieldType.convertNanosToMillis(
-                        NumericUtils.sortableBytesToLong(((PointRangeQuery) filters[i].getQuery()).getLowerPoint(), 0)
-                    );
+        RangeCollectorForPointTree collector = new RangeCollectorForPointTree(
+            incrementDocCount,
+            fieldType,
+            rangeIter,
+            maxNumNonZeroRanges,
+            activeRange
+        );
+
+        final ArrayUtil.ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(8);
+        PointValues.IntersectVisitor visitor = getIntersectVisitor(collector, comparator);
+        try {
+            intersectWithRanges(visitor, tree, collector, debugInfo);
+        } catch (CollectionTerminatedException e) {
+            logger.debug("Early terminate since no more range to collect");
+        }
+        collector.finalizePreviousRange();
+
+        return debugInfo;
+    }
+
+    private static void intersectWithRanges(
+        PointValues.IntersectVisitor visitor,
+        PointValues.PointTree pointTree,
+        RangeCollectorForPointTree collector,
+        DebugInfo debug
+    ) throws IOException {
+        PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue());
+
+        switch (r) {
+            case CELL_INSIDE_QUERY:
+                collector.countNode((int) pointTree.size());
+                debug.visitInner();
+                break;
+            case CELL_CROSSES_QUERY:
+                if (pointTree.moveToChild()) {
+                    do {
+                        intersectWithRanges(visitor, pointTree, collector, debug);
+                    } while (pointTree.moveToSibling());
+                    pointTree.moveToParent();
+                } else {
+                    pointTree.visitDocValues(visitor);
+                    debug.visitLeaf();
                 }
-                incrementDocCount.accept(bucketKey, counts[i]);
-                s++;
-                if (s > size) {
-                    break;
+                break;
+            case CELL_OUTSIDE_QUERY:
+        }
+    }
+
+    private static PointValues.IntersectVisitor getIntersectVisitor(
+        RangeCollectorForPointTree collector,
+        ArrayUtil.ByteArrayComparator comparator
+    ) {
+        return new PointValues.IntersectVisitor() {
+            @Override
+            public void visit(int docID) throws IOException {
+                // this branch should be unreachable
+                throw new UnsupportedOperationException(
+                    "This IntersectVisitor does not perform any actions on a " + "docID=" + docID + " node being visited"
+                );
+            }
+
+            @Override
+            public void visit(int docID, byte[] packedValue) throws IOException {
+                visitPoints(packedValue, collector::count);
+            }
+
+            @Override
+            public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException {
+                visitPoints(packedValue, () -> {
+                    for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
+                        collector.count();
+                    }
+                });
+            }
+
+            private void visitPoints(byte[] packedValue, CheckedRunnable<IOException> collect) throws IOException {
+                if (comparator.compare(packedValue, 0, collector.activeRangeAsByteArray[1], 0) > 0) {
+                    // need to move to next range
+                    collector.finalizePreviousRange();
+                    if (collector.iterateRangeEnd(packedValue, this::compareByteValue)) {
+                        throw new CollectionTerminatedException();
+                    }
+                }
+
+                if (pointCompare(collector.activeRangeAsByteArray[0], collector.activeRangeAsByteArray[1], packedValue)) {
+                    collect.run();
+                }
+            }
+
+            private boolean pointCompare(byte[] lower, byte[] upper, byte[] packedValue) {
+                if (compareByteValue(packedValue, lower) < 0) {
+                    return false;
                 }
+                return compareByteValue(packedValue, upper) <= 0;
+            }
+
+            private int compareByteValue(byte[] value1, byte[] value2) {
+                return comparator.compare(value1, 0, value2, 0);
             }
+
+            @Override
+            public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+                byte[] rangeMin = collector.activeRangeAsByteArray[0];
+                byte[] rangeMax = collector.activeRangeAsByteArray[1];
+
+                if (compareByteValue(rangeMax, minPackedValue) < 0) {
+                    collector.finalizePreviousRange();
+                    if (collector.iterateRangeEnd(minPackedValue, this::compareByteValue)) {
+                        throw new CollectionTerminatedException();
+                    }
+                    // compare the next range with this node's min max again
+                    // new rangeMin = previous rangeMax + 1 <= min
+                    rangeMax = collector.activeRangeAsByteArray[1];
+                }
+
+                if (compareByteValue(rangeMin, minPackedValue) > 0 || compareByteValue(rangeMax, maxPackedValue) < 0) {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                } else {
+                    return PointValues.Relation.CELL_INSIDE_QUERY;
+                }
+            }
+        };
+    }
+
+    private static class RangeCollectorForPointTree {
+        private final BiConsumer<Long, Integer> incrementDocCount;
+        private final DateFieldMapper.DateFieldType fieldType;
+        private int counter = 0;
+
+        private long[] activeRange;
+        private byte[][] activeRangeAsByteArray;
+        private final Iterator<long[]> rangeIter;
+
+        private int visitedRange = 0;
+        private final int maxNumNonZeroRange;
+
+        public RangeCollectorForPointTree(
+            BiConsumer<Long, Integer> incrementDocCount,
+            DateFieldMapper.DateFieldType fieldType,
+            Iterator<long[]> rangeIter,
+            int maxNumNonZeroRange,
+            long[] activeRange
+        ) {
+            this.incrementDocCount = incrementDocCount;
+            this.fieldType = fieldType;
+            this.rangeIter = rangeIter;
+            this.maxNumNonZeroRange = maxNumNonZeroRange;
+            this.activeRange = activeRange;
+            this.activeRangeAsByteArray = activeRangeAsByteArray();
         }
 
-        logger.debug("Fast filter optimization applied to shard {} segment {}", fastFilterContext.context.indexShard().shardId(), ctx.ord);
-        return true;
+        private void count() {
+            counter++;
+        }
+
+        private void countNode(int count) {
+            counter += count;
+        }
+
+        private void finalizePreviousRange() {
+            if (counter > 0) {
+                logger.debug("finalize previous range: {}", activeRange[0]);
+                logger.debug("counter: {}", counter);
+                incrementDocCount.accept(fieldType.convertNanosToMillis(activeRange[0]), counter);
+                counter = 0;
+            }
+        }
+
+        /**
+         * @return true when iterator exhausted or collect enough non-zero ranges
+         */
+        private boolean iterateRangeEnd(byte[] value, BiFunction<byte[], byte[], Integer> comparator) {
+            // the new value may not be contiguous to the previous one
+            // so try to find the first next range that cross the new value
+            while (comparator.apply(activeRangeAsByteArray[1], value) < 0) {
+                if (!rangeIter.hasNext()) {
+                    return true;
+                }
+                activeRange = rangeIter.next();
+                activeRangeAsByteArray = activeRangeAsByteArray();
+            }
+            visitedRange++;
+            return visitedRange > maxNumNonZeroRange;
+        }
+
+        private byte[][] activeRangeAsByteArray() {
+            byte[] lower = new byte[8];
+            byte[] upper = new byte[8];
+            NumericUtils.longToSortableBytes(activeRange[0], lower, 0);
+            NumericUtils.longToSortableBytes(activeRange[1], upper, 0);
+            return new byte[][] { lower, upper };
+        }
     }
 
-    private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leafCtx) throws IOException {
-        Weight weight = ctx.searcher().createWeight(ctx.query(), ScoreMode.COMPLETE_NO_SCORES, 1f);
-        return weight != null && weight.count(leafCtx) == leafCtx.reader().numDocs();
+    private static class DebugInfo {
+        private int leaf = 0; // leaf node visited
+        private int inner = 0; // inner node visited
+
+        private void visitLeaf() {
+            leaf++;
+        }
+
+        private void visitInner() {
+            inner++;
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
index b97c814cdf645..3713d8f83990d 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -87,6 +87,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.function.LongUnaryOperator;
 import java.util.stream.Collectors;
 
@@ -97,7 +98,7 @@
  *
  * @opensearch.internal
  */
-final class CompositeAggregator extends BucketsAggregator {
+public final class CompositeAggregator extends BucketsAggregator {
     private final int size;
     private final List<String> sourceNames;
     private final int[] reverseMuls;
@@ -171,14 +172,15 @@ final class CompositeAggregator extends BucketsAggregator {
             // bucketOrds is used for saving date histogram results
             bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), CardinalityUpperBound.ONE);
             preparedRounding = ((CompositeAggregationType) fastFilterContext.getAggregationType()).getRoundingPrepared();
-            fastFilterContext.buildFastFilter();
+            fastFilterContext.setFieldName(sourceConfigs[0].fieldType().name());
+            fastFilterContext.buildRanges();
         }
     }
 
     /**
      * Currently the filter rewrite is only supported for date histograms
      */
-    private class CompositeAggregationType extends FastFilterRewriteHelper.AbstractDateHistogramAggregationType {
+    public class CompositeAggregationType extends FastFilterRewriteHelper.AbstractDateHistogramAggregationType {
         private final RoundingValuesSource valuesSource;
         private long afterKey = -1L;
 
@@ -210,7 +212,6 @@ protected void processAfterKey(long[] bound, long interval) {
             }
         }
 
-        @Override
         public int getSize() {
             return size;
         }
@@ -706,4 +707,14 @@ private static class Entry {
             this.docIdSet = docIdSet;
         }
     }
+
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
index 12aefc540e75c..f326426800909 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
@@ -166,7 +166,8 @@ private AutoDateHistogramAggregator(
             )
         );
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
-            fastFilterContext.buildFastFilter();
+            fastFilterContext.setFieldName(valuesSourceConfig.fieldType().name());
+            fastFilterContext.buildRanges();
         }
     }
 
@@ -307,6 +308,17 @@ protected final void merge(long[] mergeMap, long newNumBuckets) {
         }
     }
 
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        super.collectDebugInfo(add);
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
+    }
+
     /**
      * Initially it uses the most fine grained rounding configuration possible
      * but as more data arrives it rebuckets the data until it "fits" in the
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index 0e830106c8284..dd4ee9196fd62 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -126,7 +126,8 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg
             )
         );
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
-            fastFilterContext.buildFastFilter();
+            fastFilterContext.setFieldName(valuesSourceConfig.fieldType().name());
+            fastFilterContext.buildRanges();
         }
     }
 
@@ -255,6 +256,12 @@ public void doClose() {
     @Override
     public void collectDebugInfo(BiConsumer<String, Object> add) {
         add.accept("total_buckets", bucketOrds.size());
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
     }
 
     /**
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
index 2a4fbca7a8541..cf95999ec5086 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -38,29 +38,42 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.TestUtil;
 import org.opensearch.common.time.DateFormatters;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
+import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.search.aggregations.AggregationBuilder;
 import org.opensearch.search.aggregations.BucketOrder;
+import org.opensearch.search.aggregations.InternalAggregation;
+import org.opensearch.search.aggregations.MultiBucketConsumerService;
 import org.opensearch.search.aggregations.bucket.terms.StringTerms;
 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.opensearch.search.aggregations.pipeline.PipelineAggregator;
 import org.opensearch.search.aggregations.support.AggregationInspectionHelper;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 
 import static java.util.stream.Collectors.toList;
+import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.hamcrest.Matchers.equalTo;
 
 public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCase {
@@ -1450,6 +1463,267 @@ private void testSearchCase(
         }
     }
 
+    public void testMultiRangeTraversal() throws IOException {
+        Map<String, Integer> dataset = new HashMap<>();
+        dataset.put("2017-02-01T09:02:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T09:59:59.999Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T10:00:00.001Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T13:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:04:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:05:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T15:59:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:48:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:59:00.000Z", randomIntBetween(100, 2000));
+
+        testFilterRewriteCase(
+            LongPoint.newRangeQuery(AGGREGABLE_DATE, asLong("2017-01-01T09:00:00.000Z"), asLong("2017-02-01T16:00:00.000Z")),
+            dataset,
+            aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L),
+            histogram -> {
+                List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                assertEquals(5, buckets.size());
+
+                Histogram.Bucket bucket = buckets.get(0);
+                assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                int expected = dataset.get("2017-02-01T09:02:00.000Z") + dataset.get("2017-02-01T09:59:59.999Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(1);
+                assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T10:00:00.001Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(2);
+                assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T13:06:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(3);
+                assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T14:04:00.000Z") + dataset.get("2017-02-01T14:05:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(4);
+                assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T15:59:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+            },
+            false,
+            collectorCount -> assertEquals(0, (int) collectorCount),
+            true
+        );
+    }
+
+    public void testMultiRangeTraversalFixedData() throws IOException {
+        Map<String, Integer> dataset = new HashMap<>();
+        dataset.put("2017-02-01T09:02:00.000Z", 512);
+        dataset.put("2017-02-01T09:59:59.999Z", 256);
+        dataset.put("2017-02-01T10:00:00.001Z", 256);
+        dataset.put("2017-02-01T13:06:00.000Z", 512);
+        dataset.put("2017-02-01T14:04:00.000Z", 256);
+        dataset.put("2017-02-01T14:05:00.000Z", 256);
+        dataset.put("2017-02-01T15:59:00.000Z", 768);
+
+        testFilterRewriteCase(
+            LongPoint.newRangeQuery(AGGREGABLE_DATE, asLong("2017-01-01T09:00:00.000Z"), asLong("2017-02-01T14:04:01.000Z")),
+            dataset,
+            aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L),
+            histogram -> {
+                List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                assertEquals(4, buckets.size());
+
+                Histogram.Bucket bucket = buckets.get(0);
+                assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                int expected = dataset.get("2017-02-01T09:02:00.000Z") + dataset.get("2017-02-01T09:59:59.999Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(1);
+                assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T10:00:00.001Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(2);
+                assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T13:06:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(3);
+                assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T14:04:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+            },
+            false,
+            collectorCount -> assertEquals(0, (int) collectorCount),
+            false
+        );
+    }
+
+    public void testMultiRangeTraversalNotApplicable() throws IOException {
+        Map<String, Integer> dataset = new HashMap<>();
+        dataset.put("2017-02-01T09:02:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T09:59:59.999Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T10:00:00.001Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T13:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:04:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:05:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T15:59:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:48:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:59:00.000Z", randomIntBetween(100, 2000));
+
+        testFilterRewriteCase(
+            new MatchAllDocsQuery(),
+            dataset,
+            aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L),
+            histogram -> {
+                List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                assertEquals(6, buckets.size());
+
+                Histogram.Bucket bucket = buckets.get(0);
+                assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                int expected = dataset.get("2017-02-01T09:02:00.000Z") + dataset.get("2017-02-01T09:59:59.999Z") + 4;
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(1);
+                assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T10:00:00.001Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(2);
+                assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T13:06:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(3);
+                assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T14:04:00.000Z") + dataset.get("2017-02-01T14:05:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(4);
+                assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T15:59:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(5);
+                assertEquals("2017-02-01T16:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T16:06:00.000Z") + dataset.get("2017-02-01T16:48:00.000Z") + dataset.get(
+                    "2017-02-01T16:59:00.000Z"
+                );
+                assertEquals(expected, bucket.getDocCount());
+            },
+            true,
+            collectCount -> assertTrue(collectCount > 0),
+            true
+        );
+    }
+
+    private void testFilterRewriteCase(
+        Query query,
+        Map<String, Integer> dataset,
+        Consumer<DateHistogramAggregationBuilder> configure,
+        Consumer<InternalDateHistogram> verify,
+        boolean useDocCountField,
+        Consumer<Integer> verifyCollectCount,
+        boolean randomWrite
+    ) throws IOException {
+        DateFieldMapper.DateFieldType fieldType = aggregableDateFieldType(false, true);
+
+        try (Directory directory = newDirectory()) {
+            if (randomWrite) {
+                try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                    Document document = new Document();
+                    if (useDocCountField) {
+                        // add the doc count field to the first document
+                        document.add(new NumericDocValuesField(DocCountFieldMapper.NAME, 5));
+                    }
+                    for (Map.Entry<String, Integer> date : dataset.entrySet()) {
+                        for (int i = 0; i < date.getValue(); i++) {
+                            long instant = asLong(date.getKey(), fieldType);
+                            document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant));
+                            document.add(new LongPoint(AGGREGABLE_DATE, instant));
+                            indexWriter.addDocument(document);
+                            document.clear();
+                        }
+                    }
+                }
+            } else {
+                // use default codec so max points in leaf is fixed to 512, to cover the node level visit and compare logic
+                try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec()))) {
+                    List<Document> documents = new ArrayList<>();
+                    for (Map.Entry<String, Integer> date : dataset.entrySet()) {
+                        for (int i = 0; i < date.getValue(); i++) {
+                            Document document = new Document();
+                            if (useDocCountField) {
+                                // add the doc count field once
+                                document.add(new NumericDocValuesField(DocCountFieldMapper.NAME, 5));
+                                useDocCountField = false;
+                            }
+                            long instant = asLong(date.getKey(), fieldType);
+                            document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant));
+                            document.add(new LongPoint(AGGREGABLE_DATE, instant));
+                            documents.add(document);
+                        }
+                    }
+                    indexWriter.addDocuments(documents);
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("_name");
+                if (configure != null) {
+                    configure.accept(aggregationBuilder);
+                }
+
+                CountingAggregator aggregator = createCountingAggregator(query, aggregationBuilder, indexSearcher, fieldType);
+                aggregator.preCollection();
+                indexSearcher.search(query, aggregator);
+                aggregator.postCollection();
+
+                MultiBucketConsumerService.MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer(
+                    Integer.MAX_VALUE,
+                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
+                );
+                InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                    aggregator.context().bigArrays(),
+                    getMockScriptService(),
+                    reduceBucketConsumer,
+                    PipelineAggregator.PipelineTree.EMPTY
+                );
+                InternalDateHistogram topLevel = (InternalDateHistogram) aggregator.buildTopLevel();
+                InternalDateHistogram histogram = (InternalDateHistogram) topLevel.reduce(Collections.singletonList(topLevel), context);
+                doAssertReducedMultiBucketConsumer(histogram, reduceBucketConsumer);
+
+                verify.accept(histogram);
+
+                verifyCollectCount.accept(aggregator.getCollectCount().get());
+            }
+        }
+    }
+
+    protected CountingAggregator createCountingAggregator(
+        Query query,
+        AggregationBuilder builder,
+        IndexSearcher searcher,
+        MappedFieldType... fieldTypes
+    ) throws IOException {
+        return new CountingAggregator(
+            new AtomicInteger(),
+            createAggregator(
+                query,
+                builder,
+                searcher,
+                new MultiBucketConsumerService.MultiBucketConsumer(
+                    DEFAULT_MAX_BUCKETS,
+                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
+                ),
+                fieldTypes
+            )
+        );
+    }
+
     private static long asLong(String dateTime) {
         return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli();
     }
diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
index e538dede07fc8..02e5d22e147d5 100644
--- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
@@ -126,7 +126,6 @@
 import org.opensearch.search.aggregations.AggregatorFactories.Builder;
 import org.opensearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.opensearch.search.aggregations.metrics.MetricsAggregator;
 import org.opensearch.search.aggregations.pipeline.PipelineAggregator;
 import org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
@@ -410,6 +409,7 @@ public boolean shouldCache(Query query) {
         );
         fieldNameToType.putAll(getFieldAliases(fieldTypes));
 
+        when(searchContext.maxAggRewriteFilters()).thenReturn(10_000);
         registerFieldTypes(searchContext, mapperService, fieldNameToType);
         doAnswer(invocation -> {
             /* Store the release-ables so we can release them at the end of the test case. This is important because aggregations don't
@@ -1123,7 +1123,7 @@ protected static class CountingAggregator extends Aggregator {
         private final AtomicInteger collectCounter;
         public final Aggregator delegate;
 
-        public CountingAggregator(AtomicInteger collectCounter, TermsAggregator delegate) {
+        public CountingAggregator(AtomicInteger collectCounter, Aggregator delegate) {
             this.collectCounter = collectCounter;
             this.delegate = delegate;
         }

From 29ebd80fdd6771ccc2023bf432977c38c76bfd55 Mon Sep 17 00:00:00 2001
From: Varun Bansal <bansvaru@amazon.com>
Date: Fri, 3 May 2024 14:11:55 +0530
Subject: [PATCH 12/52] Support wildcard/regex for indices param in
 _remotestore/_restore (#8922)

* Support wildcard/regex for indices param in _remotestore/_restore

Signed-off-by: bansvaru <bansvaru@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../remotestore/RemoteStoreRestoreIT.java     |  21 ++-
 .../opensearch/common/util/IndexUtils.java    | 128 ++++++++++++++++++
 .../recovery/RemoteStoreRestoreService.java   |   9 +-
 .../opensearch/snapshots/RestoreService.java  |   2 +-
 .../opensearch/snapshots/SnapshotUtils.java   | 105 --------------
 .../snapshots/SnapshotsService.java           |   7 +-
 .../snapshots/SnapshotUtilsTests.java         |   3 +-
 8 files changed, 152 insertions(+), 124 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/common/util/IndexUtils.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f2b92c76689f..9f013cedd82dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Tiered Caching] Expose new cache stats API ([#13237](https://github.com/opensearch-project/OpenSearch/pull/13237))
 - [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
 - Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
+- Support wildcard/regex for indices param in _remotestore/_restore ([#8922](https://github.com/opensearch-project/OpenSearch/pull/8922))
 - [Tiered Caching] Gate new stats logic behind FeatureFlags.PLUGGABLE_CACHE ([#13238](https://github.com/opensearch-project/OpenSearch/pull/13238))
 - [Tiered Caching] Add a dynamic setting to disable/enable disk cache. ([#13373](https://github.com/opensearch-project/OpenSearch/pull/13373))
 - [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
index 94acf2b1dbb27..195b2d18df7cc 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
@@ -42,7 +42,7 @@
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreRestoreIT extends BaseRemoteStoreRestoreIT {
 
     /**
@@ -295,7 +295,6 @@ public void testRestoreFlowNoRedIndex() throws Exception {
      * for multiple indices matching a wildcard name pattern.
      * @throws IOException IO Exception.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480")
     public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws Exception {
         testRestoreFlowMultipleIndices(2, true, randomIntBetween(1, 5));
     }
@@ -306,16 +305,16 @@ public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws Exce
      * with all remote-enabled red indices considered for the restore by default.
      * @throws IOException IO Exception.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480")
     public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws Exception {
         int shardCount = randomIntBetween(1, 5);
-        prepareCluster(1, 3, INDEX_NAMES, 1, shardCount);
+        int replicaCount = 1;
+        prepareCluster(1, 3, INDEX_NAMES, replicaCount, shardCount);
         String[] indices = INDEX_NAMES.split(",");
         Map<String, Map<String, Long>> indicesStats = new HashMap<>();
         for (String index : indices) {
             Map<String, Long> indexStats = indexData(2, true, index);
             indicesStats.put(index, indexStats);
-            assertEquals(shardCount, getNumShards(index).totalNumShards);
+            assertEquals(shardCount * (replicaCount + 1), getNumShards(index).totalNumShards);
         }
 
         for (String index : indices) {
@@ -337,7 +336,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws Exception
         ensureGreen(indices);
 
         for (String index : indices) {
-            assertEquals(shardCount, getNumShards(index).totalNumShards);
+            assertEquals(shardCount * (replicaCount + 1), getNumShards(index).totalNumShards);
             verifyRestoredData(indicesStats.get(index), index);
         }
     }
@@ -395,16 +394,16 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws Excep
      * except those matching the specified exclusion pattern.
      * @throws IOException IO Exception.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480")
     public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws Exception {
         int shardCount = randomIntBetween(1, 5);
-        prepareCluster(1, 3, INDEX_NAMES, 1, shardCount);
+        int replicaCount = 1;
+        prepareCluster(1, 3, INDEX_NAMES, replicaCount, shardCount);
         String[] indices = INDEX_NAMES.split(",");
         Map<String, Map<String, Long>> indicesStats = new HashMap<>();
         for (String index : indices) {
             Map<String, Long> indexStats = indexData(2, true, index);
             indicesStats.put(index, indexStats);
-            assertEquals(shardCount, getNumShards(index).totalNumShards);
+            assertEquals(shardCount * (replicaCount + 1), getNumShards(index).totalNumShards);
         }
 
         for (String index : indices) {
@@ -433,9 +432,9 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws Excep
                 PlainActionFuture.newFuture()
             );
         ensureGreen(indices[0], indices[1]);
-        assertEquals(shardCount, getNumShards(indices[0]).totalNumShards);
+        assertEquals(shardCount * (replicaCount + 1), getNumShards(indices[0]).totalNumShards);
         verifyRestoredData(indicesStats.get(indices[0]), indices[0]);
-        assertEquals(shardCount, getNumShards(indices[1]).totalNumShards);
+        assertEquals(shardCount * (replicaCount + 1), getNumShards(indices[1]).totalNumShards);
         verifyRestoredData(indicesStats.get(indices[1]), indices[1]);
         ensureRed(indices[2], indices[3]);
     }
diff --git a/server/src/main/java/org/opensearch/common/util/IndexUtils.java b/server/src/main/java/org/opensearch/common/util/IndexUtils.java
new file mode 100644
index 0000000000000..b4f049617c4fc
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/util/IndexUtils.java
@@ -0,0 +1,128 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.util;
+
+import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.common.regex.Regex;
+import org.opensearch.index.IndexNotFoundException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Common Utility methods for Indices.
+ *
+ * @opensearch.internal
+ */
+public class IndexUtils {
+
+    /**
+     * Filters out list of available indices based on the list of selected indices.
+     *
+     * @param availableIndices list of available indices
+     * @param selectedIndices  list of selected indices
+     * @param indicesOptions    ignore indices flag
+     * @return filtered out indices
+     */
+    public static List<String> filterIndices(List<String> availableIndices, String[] selectedIndices, IndicesOptions indicesOptions) {
+        if (IndexNameExpressionResolver.isAllIndices(Arrays.asList(selectedIndices))) {
+            return availableIndices;
+        }
+
+        // Move the exclusions to end of list to ensure they are processed
+        // after explicitly selected indices are chosen.
+        final List<String> excludesAtEndSelectedIndices = Stream.concat(
+            Arrays.stream(selectedIndices).filter(s -> s.isEmpty() || s.charAt(0) != '-'),
+            Arrays.stream(selectedIndices).filter(s -> !s.isEmpty() && s.charAt(0) == '-')
+        ).collect(Collectors.toUnmodifiableList());
+
+        Set<String> result = null;
+        for (int i = 0; i < excludesAtEndSelectedIndices.size(); i++) {
+            String indexOrPattern = excludesAtEndSelectedIndices.get(i);
+            boolean add = true;
+            if (!indexOrPattern.isEmpty()) {
+                if (availableIndices.contains(indexOrPattern)) {
+                    if (result == null) {
+                        result = new HashSet<>();
+                    }
+                    result.add(indexOrPattern);
+                    continue;
+                }
+                if (indexOrPattern.charAt(0) == '+') {
+                    add = true;
+                    indexOrPattern = indexOrPattern.substring(1);
+                    // if its the first, add empty set
+                    if (i == 0) {
+                        result = new HashSet<>();
+                    }
+                } else if (indexOrPattern.charAt(0) == '-') {
+                    // If the first index pattern is an exclusion, then all patterns are exclusions due to the
+                    // reordering logic above. In this case, the request is interpreted as "include all indexes except
+                    // those matching the exclusions" so we add all indices here and then remove the ones that match the exclusion patterns.
+                    if (i == 0) {
+                        result = new HashSet<>(availableIndices);
+                    }
+                    add = false;
+                    indexOrPattern = indexOrPattern.substring(1);
+                }
+            }
+            if (indexOrPattern.isEmpty() || !Regex.isSimpleMatchPattern(indexOrPattern)) {
+                if (!availableIndices.contains(indexOrPattern)) {
+                    if (!indicesOptions.ignoreUnavailable()) {
+                        throw new IndexNotFoundException(indexOrPattern);
+                    } else {
+                        if (result == null) {
+                            // add all the previous ones...
+                            result = new HashSet<>(availableIndices.subList(0, i));
+                        }
+                    }
+                } else {
+                    if (result != null) {
+                        if (add) {
+                            result.add(indexOrPattern);
+                        } else {
+                            result.remove(indexOrPattern);
+                        }
+                    }
+                }
+                continue;
+            }
+            if (result == null) {
+                // add all the previous ones...
+                result = new HashSet<>(availableIndices.subList(0, i));
+            }
+            boolean found = false;
+            for (String index : availableIndices) {
+                if (Regex.simpleMatch(indexOrPattern, index)) {
+                    found = true;
+                    if (add) {
+                        result.add(index);
+                    } else {
+                        result.remove(index);
+                    }
+                }
+            }
+            if (!found && !indicesOptions.allowNoIndices()) {
+                throw new IndexNotFoundException(indexOrPattern);
+            }
+        }
+        if (result == null) {
+            return Collections.unmodifiableList(new ArrayList<>(Arrays.asList(selectedIndices)));
+        }
+        return Collections.unmodifiableList(new ArrayList<>(result));
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index 23bb4cea17a20..fe90f24b0f544 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
+import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateUpdateTask;
 import org.opensearch.cluster.block.ClusterBlocks;
@@ -48,6 +49,7 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING;
 
 /**
@@ -158,7 +160,12 @@ public RemoteRestoreResult restore(
                 throw new IllegalStateException("Unable to restore remote index metadata", e);
             }
         } else {
-            for (String indexName : indexNames) {
+            List<String> filteredIndices = filterIndices(
+                List.of(currentState.metadata().getConcreteAllIndices()),
+                indexNames,
+                IndicesOptions.fromOptions(true, true, true, true)
+            );
+            for (String indexName : filteredIndices) {
                 IndexMetadata indexMetadata = currentState.metadata().index(indexName);
                 if (indexMetadata == null) {
                     logger.warn("Index restore is not supported for non-existent index. Skipping: {}", indexName);
diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java
index e6a6b747c2baf..5883a8a37be71 100644
--- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java
@@ -125,12 +125,12 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
 import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.opensearch.common.util.set.Sets.newHashSet;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
 import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING;
-import static org.opensearch.snapshots.SnapshotUtils.filterIndices;
 
 /**
  * Service responsible for restoring snapshots
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java b/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java
index e7338a29cafeb..8cce5e1c98254 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java
@@ -31,24 +31,15 @@
 
 package org.opensearch.snapshots;
 
-import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
-import org.opensearch.common.regex.Regex;
 import org.opensearch.index.IndexModule;
-import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexSettings;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Snapshot utilities
@@ -57,102 +48,6 @@
  */
 public class SnapshotUtils {
 
-    /**
-     * Filters out list of available indices based on the list of selected indices.
-     *
-     * @param availableIndices list of available indices
-     * @param selectedIndices  list of selected indices
-     * @param indicesOptions    ignore indices flag
-     * @return filtered out indices
-     */
-    public static List<String> filterIndices(List<String> availableIndices, String[] selectedIndices, IndicesOptions indicesOptions) {
-        if (IndexNameExpressionResolver.isAllIndices(Arrays.asList(selectedIndices))) {
-            return availableIndices;
-        }
-
-        // Move the exclusions to end of list to ensure they are processed
-        // after explicitly selected indices are chosen.
-        final List<String> excludesAtEndSelectedIndices = Stream.concat(
-            Arrays.stream(selectedIndices).filter(s -> s.isEmpty() || s.charAt(0) != '-'),
-            Arrays.stream(selectedIndices).filter(s -> !s.isEmpty() && s.charAt(0) == '-')
-        ).collect(Collectors.toUnmodifiableList());
-
-        Set<String> result = null;
-        for (int i = 0; i < excludesAtEndSelectedIndices.size(); i++) {
-            String indexOrPattern = excludesAtEndSelectedIndices.get(i);
-            boolean add = true;
-            if (!indexOrPattern.isEmpty()) {
-                if (availableIndices.contains(indexOrPattern)) {
-                    if (result == null) {
-                        result = new HashSet<>();
-                    }
-                    result.add(indexOrPattern);
-                    continue;
-                }
-                if (indexOrPattern.charAt(0) == '+') {
-                    add = true;
-                    indexOrPattern = indexOrPattern.substring(1);
-                    // if its the first, add empty set
-                    if (i == 0) {
-                        result = new HashSet<>();
-                    }
-                } else if (indexOrPattern.charAt(0) == '-') {
-                    // If the first index pattern is an exclusion, then all patterns are exclusions due to the
-                    // reordering logic above. In this case, the request is interpreted as "include all indexes except
-                    // those matching the exclusions" so we add all indices here and then remove the ones that match the exclusion patterns.
-                    if (i == 0) {
-                        result = new HashSet<>(availableIndices);
-                    }
-                    add = false;
-                    indexOrPattern = indexOrPattern.substring(1);
-                }
-            }
-            if (indexOrPattern.isEmpty() || !Regex.isSimpleMatchPattern(indexOrPattern)) {
-                if (!availableIndices.contains(indexOrPattern)) {
-                    if (!indicesOptions.ignoreUnavailable()) {
-                        throw new IndexNotFoundException(indexOrPattern);
-                    } else {
-                        if (result == null) {
-                            // add all the previous ones...
-                            result = new HashSet<>(availableIndices.subList(0, i));
-                        }
-                    }
-                } else {
-                    if (result != null) {
-                        if (add) {
-                            result.add(indexOrPattern);
-                        } else {
-                            result.remove(indexOrPattern);
-                        }
-                    }
-                }
-                continue;
-            }
-            if (result == null) {
-                // add all the previous ones...
-                result = new HashSet<>(availableIndices.subList(0, i));
-            }
-            boolean found = false;
-            for (String index : availableIndices) {
-                if (Regex.simpleMatch(indexOrPattern, index)) {
-                    found = true;
-                    if (add) {
-                        result.add(index);
-                    } else {
-                        result.remove(index);
-                    }
-                }
-            }
-            if (!found && !indicesOptions.allowNoIndices()) {
-                throw new IndexNotFoundException(indexOrPattern);
-            }
-        }
-        if (result == null) {
-            return Collections.unmodifiableList(new ArrayList<>(Arrays.asList(selectedIndices)));
-        }
-        return Collections.unmodifiableList(new ArrayList<>(result));
-    }
-
     /**
      * Validates if there are any remote snapshots backing an index
      * @param metadata index metadata from cluster state
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
index 22b640963e896..acc2dc83749cd 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
@@ -131,6 +131,7 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableList;
 import static org.opensearch.cluster.SnapshotsInProgress.completed;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY;
@@ -475,11 +476,7 @@ public ClusterState execute(ClusterState currentState) {
                         indicesForSnapshot.add(indexId.getName());
                     }
                 }
-                final List<String> matchingIndices = SnapshotUtils.filterIndices(
-                    indicesForSnapshot,
-                    request.indices(),
-                    request.indicesOptions()
-                );
+                final List<String> matchingIndices = filterIndices(indicesForSnapshot, request.indices(), request.indicesOptions());
                 if (matchingIndices.isEmpty()) {
                     throw new SnapshotException(
                         new Snapshot(repositoryName, sourceSnapshotId),
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java
index 14e711e03a345..871f2cdd88ea1 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java
@@ -46,6 +46,7 @@
 import java.util.Map;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 
 public class SnapshotUtilsTests extends OpenSearchTestCase {
@@ -94,7 +95,7 @@ private void assertIndexNameFiltering(String[] indices, String[] filter, String[
 
     private void assertIndexNameFiltering(String[] indices, String[] filter, IndicesOptions indicesOptions, String[] expected) {
         List<String> indicesList = Arrays.asList(indices);
-        List<String> actual = SnapshotUtils.filterIndices(indicesList, filter, indicesOptions);
+        List<String> actual = filterIndices(indicesList, filter, indicesOptions);
         assertThat(actual, containsInAnyOrder(expected));
     }
 

From 89c6c27f7a2f303626e06ab347dc992a3f697adf Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Fri, 3 May 2024 15:30:57 +0530
Subject: [PATCH 13/52] Clear transient cluster setting at the end of the test
 (#13508)

Remove check of non zero assertion of time as it is non deterministic
Signed-off-by: Gaurav Bafna <gbbafna@amazon.com>
---
 .../action/admin/indices/create/RemoteCloneIndexIT.java       | 4 +++-
 .../index/translog/transfer/TranslogTransferManagerTests.java | 4 +---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
index 98c2a3a1581b8..4be049c9a9109 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
@@ -129,7 +129,9 @@ public void testCreateCloneIndex() {
                 .cluster()
                 .prepareUpdateSettings()
                 .setTransientSettings(
-                    Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                    Settings.builder()
+                        .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                        .put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), (String) null)
                 )
                 .get();
         }
diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
index 81ae479d018b0..8b3fc6651a505 100644
--- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
@@ -582,9 +582,7 @@ private void assertNoDownloadStats(boolean nonZeroUploadTime) {
         assertEquals(0, remoteTranslogTransferTracker.getDownloadBytesSucceeded());
         assertEquals(0, remoteTranslogTransferTracker.getTotalDownloadsSucceeded());
         assertEquals(0, remoteTranslogTransferTracker.getLastSuccessfulDownloadTimestamp());
-        if (nonZeroUploadTime) {
-            assertNotEquals(0, remoteTranslogTransferTracker.getTotalDownloadTimeInMillis());
-        } else {
+        if (nonZeroUploadTime == false) {
             assertEquals(0, remoteTranslogTransferTracker.getTotalDownloadTimeInMillis());
         }
     }

From 14cd0e33ebfe6c84fcb09687c060e5a0f047c0a7 Mon Sep 17 00:00:00 2001
From: samuel-oci <97131656+samuel-oci@users.noreply.github.com>
Date: Fri, 3 May 2024 09:31:01 -0700
Subject: [PATCH 14/52] FEATURE-13448: add use compound file setting (#13478)

* add use compound file setting

Signed-off-by: Samuel Herman <sherman8915@gmail.com>

* review feedback

Signed-off-by: Samuel Herman <sherman8915@gmail.com>

---------

Signed-off-by: Samuel Herman <sherman8915@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 .../common/settings/IndexScopedSettings.java  |  1 +
 .../opensearch/index/engine/EngineConfig.java | 10 ++++
 .../index/engine/InternalEngine.java          |  2 +-
 .../index/engine/EngineConfigTests.java       |  7 +++
 .../index/engine/InternalEngineTests.java     | 52 +++++++++++++++++++
 6 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f013cedd82dc..a49ba7907c79d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
 - Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285))
 - Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
 - Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index c6c312d6b6eea..980c432774f6e 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -198,6 +198,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 EngineConfig.INDEX_CODEC_SETTING,
                 EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING,
                 EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,
+                EngineConfig.INDEX_USE_COMPOUND_FILE,
                 IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS,
                 IndexSettings.DEFAULT_PIPELINE,
                 IndexSettings.FINAL_PIPELINE,
diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
index 8106b65bddeec..fe2ce470cda03 100644
--- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
+++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
@@ -236,6 +236,12 @@ private static void doValidateCodecSettings(final String codec) {
         Property.Dynamic
     );
 
+    public static final Setting<Boolean> INDEX_USE_COMPOUND_FILE = Setting.boolSetting(
+        "index.use_compound_file",
+        true,
+        Property.IndexScope
+    );
+
     private final TranslogConfig translogConfig;
 
     private final TranslogFactory translogFactory;
@@ -494,6 +500,10 @@ public boolean isReadOnlyReplica() {
         return indexSettings.isSegRepEnabledOrRemoteNode() && isReadOnlyReplica;
     }
 
+    public boolean useCompoundFile() {
+        return indexSettings.getValue(INDEX_USE_COMPOUND_FILE);
+    }
+
     /**
      * Returns the underlying startedPrimarySupplier.
      * @return the primary mode supplier.
diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
index 7bacec22fc850..59165b936aec8 100644
--- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -2341,7 +2341,7 @@ private IndexWriterConfig getIndexWriterConfig() {
         iwc.setSimilarity(engineConfig.getSimilarity());
         iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
         iwc.setCodec(engineConfig.getCodec());
-        iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
+        iwc.setUseCompoundFile(engineConfig.useCompoundFile());
         if (config().getIndexSort() != null) {
             iwc.setIndexSort(config().getIndexSort());
         }
diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java
index 9197b959a54f1..f42188d55bca3 100644
--- a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java
@@ -32,6 +32,13 @@ public void setUp() throws Exception {
         defaultIndexSettings = IndexSettingsModule.newIndexSettings("test", defaultIndexMetadata.getSettings());
     }
 
+    public void testEngineConfig_DefaultValueFoUseCompoundFile() {
+        EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings)
+            .retentionLeasesSupplier(() -> RetentionLeases.EMPTY)
+            .build();
+        assertTrue(config.useCompoundFile());
+    }
+
     public void testEngineConfig_DefaultValueForReadOnlyEngine() {
         EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings)
             .retentionLeasesSupplier(() -> RetentionLeases.EMPTY)
diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
index 74aef54987842..54a562642d4ab 100644
--- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
@@ -342,6 +342,58 @@ public void testVerboseSegments() throws Exception {
         }
     }
 
+    public void testSegmentsWithUseCompoundFileFlag_true() throws IOException {
+        try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
+            ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
+            engine.index(index);
+            engine.flush();
+            final List<Segment> segments = engine.segments(false);
+            assertThat(segments, hasSize(1));
+            assertTrue(segments.get(0).compound);
+            boolean cfeCompoundFileFound = false;
+            boolean cfsCompoundFileFound = false;
+            for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) {
+                if (fileName.endsWith(".cfe")) {
+                    cfeCompoundFileFound = true;
+                }
+                if (fileName.endsWith(".cfs")) {
+                    cfsCompoundFileFound = true;
+                }
+            }
+            Assert.assertTrue(cfeCompoundFileFound);
+            Assert.assertTrue(cfsCompoundFileFound);
+        }
+    }
+
+    public void testSegmentsWithUseCompoundFileFlag_false() throws IOException {
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
+            "test",
+            Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_USE_COMPOUND_FILE.getKey(), false).build()
+        );
+        try (Store store = createStore(); Engine engine = createEngine(indexSettings, store, createTempDir(), new TieredMergePolicy())) {
+            ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
+            engine.index(index);
+            engine.flush();
+            final List<Segment> segments = engine.segments(false);
+            assertThat(segments, hasSize(1));
+            assertFalse(segments.get(0).compound);
+            boolean cfeCompoundFileFound = false;
+            boolean cfsCompoundFileFound = false;
+            for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) {
+                if (fileName.endsWith(".cfe")) {
+                    cfeCompoundFileFound = true;
+                }
+                if (fileName.endsWith(".cfs")) {
+                    cfsCompoundFileFound = true;
+                }
+            }
+            Assert.assertFalse(cfeCompoundFileFound);
+            Assert.assertFalse(cfsCompoundFileFound);
+        }
+    }
+
     public void testSegmentsWithMergeFlag() throws Exception {
         try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
             ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);

From 0c1cd75f66d09ccad01b810f1bdf8f0841ed1ef4 Mon Sep 17 00:00:00 2001
From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com>
Date: Fri, 3 May 2024 23:33:42 +0530
Subject: [PATCH 15/52] Update Cluster Manager Node Timeout to 3s for flaky
 test testHealthOnClusterManagerFailover (#13505)

Signed-off-by: Rahul Karajgikar <karajgik@amazon.com>
Co-authored-by: Rahul Karajgikar <karajgik@amazon.com>
---
 .../java/org/opensearch/cluster/ClusterHealthIT.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
index d63b87cbee6f7..0304e00a49070 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
@@ -377,7 +377,7 @@ public void testHealthOnClusterManagerFailover() throws Exception {
                     .prepareHealth()
                     .setWaitForEvents(Priority.LANGUID)
                     .setWaitForGreenStatus()
-                    .setClusterManagerNodeTimeout(TimeValue.timeValueMinutes(2))
+                    .setClusterManagerNodeTimeout(TimeValue.timeValueMinutes(3))
                     .execute()
             );
             internalCluster().restartNode(internalCluster().getClusterManagerName(), InternalTestCluster.EMPTY_CALLBACK);

From 91067134e8e0a6dbfb05317afe4752194a8a69d2 Mon Sep 17 00:00:00 2001
From: Andriy Redko <andriy.redko@aiven.io>
Date: Fri, 3 May 2024 21:06:34 -0400
Subject: [PATCH 16/52] DATETIME_FORMATTER_CACHING_SETTING experimental feature
 should not default to 'true' (#13532)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                  |  1 +
 .../opensearch/common/util/FeatureFlags.java  |  2 +-
 .../common/util/FeatureFlagTests.java         | 12 ++---
 .../index/mapper/DateFieldMapperTests.java    | 18 ++++++++
 .../index/mapper/RangeFieldMapperTests.java   | 26 +++++++++++
 .../index/mapper/RangeFieldTypeTests.java     | 45 +++++++++++++++++++
 6 files changed, 94 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a49ba7907c79d..fe358a240f110 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -97,6 +97,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Improve the error messages for _stats with closed indices ([#13012](https://github.com/opensearch-project/OpenSearch/pull/13012))
 - Ignore BaseRestHandler unconsumed content check as it's always consumed. ([#13290](https://github.com/opensearch-project/OpenSearch/pull/13290))
 - Fix mapper_parsing_exception when using flat_object fields with names longer than 11 characters ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13259))
+- DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index bdfce72d106d3..a72583607ede0 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -81,7 +81,7 @@ public class FeatureFlags {
 
     public static final Setting<Boolean> DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting(
         DATETIME_FORMATTER_CACHING,
-        true,
+        false,
         Property.NodeScope
     );
 
diff --git a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
index 88cb3782252b7..6b6c7b96266d3 100644
--- a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
+++ b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
@@ -39,12 +39,6 @@ public void testNonBooleanFeatureFlag() {
         assertFalse(FeatureFlags.isEnabled(javaVersionProperty));
     }
 
-    public void testBooleanFeatureFlagWithDefaultSetToTrue() {
-        final String testFlag = DATETIME_FORMATTER_CACHING;
-        assertNotNull(testFlag);
-        assertTrue(FeatureFlags.isEnabled(testFlag));
-    }
-
     public void testBooleanFeatureFlagWithDefaultSetToFalse() {
         final String testFlag = IDENTITY;
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
@@ -52,17 +46,17 @@ public void testBooleanFeatureFlagWithDefaultSetToFalse() {
         assertFalse(FeatureFlags.isEnabled(testFlag));
     }
 
-    public void testBooleanFeatureFlagInitializedWithEmptySettingsAndDefaultSetToTrue() {
+    public void testBooleanFeatureFlagInitializedWithEmptySettingsAndDefaultSetToFalse() {
         final String testFlag = DATETIME_FORMATTER_CACHING;
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
         assertNotNull(testFlag);
-        assertTrue(FeatureFlags.isEnabled(testFlag));
+        assertFalse(FeatureFlags.isEnabled(testFlag));
     }
 
     public void testInitializeFeatureFlagsWithExperimentalSettings() {
         FeatureFlags.initializeFeatureFlags(Settings.builder().put(IDENTITY, true).build());
         assertTrue(FeatureFlags.isEnabled(IDENTITY));
-        assertTrue(FeatureFlags.isEnabled(DATETIME_FORMATTER_CACHING));
+        assertFalse(FeatureFlags.isEnabled(DATETIME_FORMATTER_CACHING));
         assertFalse(FeatureFlags.isEnabled(EXTENSIONS));
         // reset FeatureFlags to defaults
         FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
index 054d3956596af..2aa310ae959d9 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.opensearch.common.time.DateFormatter;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.termvectors.TermVectorsService;
 import org.opensearch.search.DocValueFormat;
@@ -45,8 +46,10 @@
 import java.time.ZonedDateTime;
 import java.util.List;
 
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assume.assumeThat;
 
 public class DateFieldMapperTests extends MapperTestCase {
 
@@ -146,7 +149,22 @@ public void testStore() throws Exception {
         assertEquals(1457654400000L, storedField.numericValue().longValue());
     }
 
+    public void testIgnoreMalformedLegacy() throws IOException {
+        assumeThat("Using legacy datetime format as default", FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING), is(false));
+        testIgnoreMalformedForValue(
+            "2016-03-99",
+            "failed to parse date field [2016-03-99] with format [strict_date_optional_time||epoch_millis]"
+        );
+        testIgnoreMalformedForValue("-2147483648", "Invalid value for Year (valid values -999999999 - 999999999): -2147483648");
+        testIgnoreMalformedForValue("-522000000", "long overflow");
+    }
+
     public void testIgnoreMalformed() throws IOException {
+        assumeThat(
+            "Using experimental datetime format as default",
+            FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING),
+            is(true)
+        );
         testIgnoreMalformedForValue(
             "2016-03-99",
             "failed to parse date field [2016-03-99] with format [strict_date_time_no_millis||strict_date_optional_time||epoch_millis]"
diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java
index 331bfb7b2ddf4..91eab942c499a 100644
--- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.network.InetAddresses;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -51,8 +52,10 @@
 import static org.opensearch.index.query.RangeQueryBuilder.GT_FIELD;
 import static org.opensearch.index.query.RangeQueryBuilder.LTE_FIELD;
 import static org.opensearch.index.query.RangeQueryBuilder.LT_FIELD;
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assume.assumeThat;
 
 public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
     private static final String FROM_DATE = "2016-10-31";
@@ -351,7 +354,30 @@ public void testIllegalArguments() throws Exception {
         assertThat(e.getMessage(), containsString("should not define a dateTimeFormatter"));
     }
 
+    public void testSerializeDefaultsLegacy() throws Exception {
+        assumeThat("Using legacy datetime format as default", FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING), is(false));
+
+        for (String type : types()) {
+            DocumentMapper docMapper = createDocumentMapper(fieldMapping(b -> b.field("type", type)));
+            RangeFieldMapper mapper = (RangeFieldMapper) docMapper.root().getMapper("field");
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS);
+            String got = builder.endObject().toString();
+
+            // if type is date_range we check that the mapper contains the default format and locale
+            // otherwise it should not contain a locale or format
+            assertTrue(got, got.contains("\"format\":\"strict_date_optional_time||epoch_millis\"") == type.equals("date_range"));
+            assertTrue(got, got.contains("\"locale\":" + "\"" + Locale.ROOT + "\"") == type.equals("date_range"));
+        }
+    }
+
     public void testSerializeDefaults() throws Exception {
+        assumeThat(
+            "Using experimental datetime format as default",
+            FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING),
+            is(true)
+        );
+
         for (String type : types()) {
             DocumentMapper docMapper = createDocumentMapper(fieldMapping(b -> b.field("type", type)));
             RangeFieldMapper mapper = (RangeFieldMapper) docMapper.root().getMapper("field");
diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java
index 00b48240d0567..49bf227e5073c 100644
--- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java
@@ -51,6 +51,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.DateFieldMapper.DateFieldType;
 import org.opensearch.index.mapper.RangeFieldMapper.RangeFieldType;
@@ -65,8 +66,10 @@
 import java.util.Collections;
 import java.util.Map;
 
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assume.assumeThat;
 
 public class RangeFieldTypeTests extends FieldTypeTestCase {
     RangeType type;
@@ -249,7 +252,49 @@ private QueryShardContext createContext() {
         );
     }
 
+    public void testDateRangeQueryUsingMappingFormatLegacy() {
+        assumeThat("Using legacy datetime format as default", FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING), is(false));
+
+        QueryShardContext context = createContext();
+        RangeFieldType strict = new RangeFieldType("field", RangeFieldMapper.Defaults.DATE_FORMATTER);
+        // don't use DISJOINT here because it doesn't work on date fields which we want to compare bounds with
+        ShapeRelation relation = randomValueOtherThan(ShapeRelation.DISJOINT, () -> randomFrom(ShapeRelation.values()));
+
+        // dates will break the default format, month/day of month is turned around in the format
+        final String from = "2016-15-06T15:29:50+08:00";
+        final String to = "2016-16-06T15:29:50+08:00";
+
+        OpenSearchParseException ex = expectThrows(
+            OpenSearchParseException.class,
+            () -> strict.rangeQuery(from, to, true, true, relation, null, null, context)
+        );
+        assertThat(
+            ex.getMessage(),
+            containsString("failed to parse date field [2016-15-06T15:29:50+08:00] with format [strict_date_optional_time||epoch_millis]")
+        );
+
+        // setting mapping format which is compatible with those dates
+        final DateFormatter formatter = DateFormatter.forPattern("yyyy-dd-MM'T'HH:mm:ssZZZZZ");
+        assertEquals(1465975790000L, formatter.parseMillis(from));
+        assertEquals(1466062190000L, formatter.parseMillis(to));
+
+        RangeFieldType fieldType = new RangeFieldType("field", formatter);
+        final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context);
+        assertEquals("field:<ranges:[1465975790000 : 1466062190999]>", ((IndexOrDocValuesQuery) query).getIndexQuery().toString());
+
+        // compare lower and upper bounds with what we would get on a `date` field
+        DateFieldType dateFieldType = new DateFieldType("field", DateFieldMapper.Resolution.MILLISECONDS, formatter);
+        final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context);
+        assertEquals("field:[1465975790000 TO 1466062190999]", ((IndexOrDocValuesQuery) queryOnDateField).getIndexQuery().toString());
+    }
+
     public void testDateRangeQueryUsingMappingFormat() {
+        assumeThat(
+            "Using experimental datetime format as default",
+            FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING),
+            is(true)
+        );
+
         QueryShardContext context = createContext();
         RangeFieldType strict = new RangeFieldType("field", RangeFieldMapper.Defaults.DATE_FORMATTER);
         // don't use DISJOINT here because it doesn't work on date fields which we want to compare bounds with

From 7c989546fe7c0ff6943d5b6bfbcfed9e3b92122b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 10:39:15 -0400
Subject: [PATCH 17/52] Bump com.github.spullara.mustache.java:compiler from
 0.9.11 to 0.9.13 in /modules/lang-mustache (#13559)

* Bump com.github.spullara.mustache.java:compiler

Bumps [com.github.spullara.mustache.java:compiler](https://github.com/spullara/mustache.java) from 0.9.11 to 0.9.13.
- [Commits](https://github.com/spullara/mustache.java/compare/mustache.java-0.9.11...mustache.java-0.9.13)

---
updated-dependencies:
- dependency-name: com.github.spullara.mustache.java:compiler
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                            | 2 +-
 modules/lang-mustache/build.gradle                      | 2 +-
 modules/lang-mustache/licenses/compiler-0.9.11.jar.sha1 | 1 -
 modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1 | 1 +
 4 files changed, 3 insertions(+), 3 deletions(-)
 delete mode 100644 modules/lang-mustache/licenses/compiler-0.9.11.jar.sha1
 create mode 100644 modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe358a240f110..37d0eb31133df 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,7 +57,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update google dependencies in repository-gcs and discovery-gce ([#13213](https://github.com/opensearch-project/OpenSearch/pull/13213))
 - Bump `com.google.apis:google-api-services-compute` from v1-rev235-1.25.0 to v1-rev20240407-2.0.0 ([#13333](https://github.com/opensearch-project/OpenSearch/pull/13333))
 - Bump `commons-cli:commons-cli` from 1.6.0 to 1.7.0 ([#13331](https://github.com/opensearch-project/OpenSearch/pull/13331))
-- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.11 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329))
+- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
 - Bump `jakarta.enterprise:jakarta.enterprise.cdi-api` from 4.0.1 to 4.1.0 ([#13328](https://github.com/opensearch-project/OpenSearch/pull/13328))
 - Bump `com.google.api.grpc:proto-google-iam-v1` from 0.12.0 to 1.33.0 ([#13332](https://github.com/opensearch-project/OpenSearch/pull/13332))
 - OpenJDK Update (April 2024 Patch releases), update to Eclipse Temurin 21.0.3+9 ([#13389](https://github.com/opensearch-project/OpenSearch/pull/13389))
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 4bf75b593ebf2..bcf5c07ea8c64 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -38,7 +38,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api "com.github.spullara.mustache.java:compiler:0.9.11"
+  api "com.github.spullara.mustache.java:compiler:0.9.13"
 }
 
 restResources {
diff --git a/modules/lang-mustache/licenses/compiler-0.9.11.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.11.jar.sha1
deleted file mode 100644
index a77675488b2e0..0000000000000
--- a/modules/lang-mustache/licenses/compiler-0.9.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1bce858aca4f0ce93fdb939de8c8474431c06322
\ No newline at end of file
diff --git a/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
new file mode 100644
index 0000000000000..70d53aac260eb
--- /dev/null
+++ b/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
@@ -0,0 +1 @@
+60666500a7dce7a5d3e17c09b46ea6f037192bd5
\ No newline at end of file

From 7c5739bab60f75bb54cfc88f993c985f0353d9c6 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <31575408+shiv0408@users.noreply.github.com>
Date: Mon, 6 May 2024 21:11:47 +0530
Subject: [PATCH 18/52] Update version for batch shard allocation to 2.14
 (#13469)

* Update version for batch shard allocation to 2.14

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
---
 .../cluster/routing/allocation/AllocationService.java         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
index d6b364887b560..71e562253bf58 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
@@ -569,7 +569,9 @@ private void allocateExistingUnassignedShards(RoutingAllocation allocation) {
          Use batch mode if enabled and there is no custom allocator set for Allocation service
          */
         Boolean batchModeEnabled = EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(settings);
-        if (batchModeEnabled && allocation.nodes().getMinNodeVersion().onOrAfter(Version.V_3_0_0) && existingShardsAllocators.size() == 2) {
+        if (batchModeEnabled
+            && allocation.nodes().getMinNodeVersion().onOrAfter(Version.V_2_14_0)
+            && existingShardsAllocators.size() == 2) {
             /*
              If we do not have any custom allocator set then we will be using ShardsBatchGatewayAllocator
              Currently AllocationService will not run any custom Allocator that implements allocateAllUnassignedShards

From bd005185104d2783a65e3faa5c0cf505b3a2f2f8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 12:58:08 -0400
Subject: [PATCH 19/52] Bump org.apache.commons:commons-text from 1.11.0 to
 1.12.0 in /test/fixtures/hdfs-fixture (#13557)

* Bump org.apache.commons:commons-text in /test/fixtures/hdfs-fixture

Bumps org.apache.commons:commons-text from 1.11.0 to 1.12.0.

---
updated-dependencies:
- dependency-name: org.apache.commons:commons-text
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                            | 1 +
 test/fixtures/hdfs-fixture/build.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37d0eb31133df..752794e06319f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.bouncycastle:bc-fips` from 1.0.2.4 to 1.0.2.5 ([#13446](https://github.com/opensearch-project/OpenSearch/pull/13446))
 - Bump `lycheeverse/lychee-action` from 1.9.3 to 1.10.0 ([#13447](https://github.com/opensearch-project/OpenSearch/pull/13447))
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
+- Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
 
 ### Changed
 - [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 3eb026b19e20c..8ac13d897e0b5 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -69,7 +69,7 @@ dependencies {
   api "org.eclipse.jetty:jetty-server:${versions.jetty}"
   api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}"
   api 'org.apache.zookeeper:zookeeper:3.9.2'
-  api "org.apache.commons:commons-text:1.11.0"
+  api "org.apache.commons:commons-text:1.12.0"
   api "commons-net:commons-net:3.10.0"
   api "ch.qos.logback:logback-core:1.5.3"
   api "ch.qos.logback:logback-classic:1.2.13"

From a752608d8117aa72d1bc40b55456b6447309f365 Mon Sep 17 00:00:00 2001
From: Sagar <99425694+sgup432@users.noreply.github.com>
Date: Mon, 6 May 2024 17:25:45 -0700
Subject: [PATCH 20/52] Fix negative requestStats memory_size issue (#13553)

This solves the bug where RequestStats memory_size metric was going negative in certain scenarios
as reported in the issue. It turns out that the issue occurs when an indexShard is deleted and then
reallocated on the same node. So whenever stale entries from older shard are deleted, those are
accounted for the new shard which has the same shardId.

---------

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 .../indices/IndicesRequestCacheIT.java        | 105 ++++++++++++
 .../cache/request/ShardRequestCache.java      |  25 +--
 .../indices/IndicesRequestCache.java          |  34 +++-
 .../IRCKeyWriteableSerializerTests.java       |   3 +-
 .../indices/IndicesRequestCacheTests.java     | 157 ++++++++++++++++--
 6 files changed, 291 insertions(+), 34 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 752794e06319f..43975a79e9cd7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -99,6 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Ignore BaseRestHandler unconsumed content check as it's always consumed. ([#13290](https://github.com/opensearch-project/OpenSearch/pull/13290))
 - Fix mapper_parsing_exception when using flat_object fields with names longer than 11 characters ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13259))
 - DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
+- Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 
 ### Security
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index b23aac08702df..ae2295cb874f5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -34,6 +34,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
@@ -43,11 +44,17 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchType;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.env.NodeEnvironment;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
@@ -59,6 +66,8 @@
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.hamcrest.OpenSearchAssertions;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -70,6 +79,7 @@
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
 import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
@@ -1240,6 +1250,101 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
         }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
     }
 
+    public void testDeleteAndCreateSameIndexShardOnSameNode() throws Exception {
+        String node_1 = internalCluster().startNode(Settings.builder().build());
+        Client client = client(node_1);
+
+        logger.info("Starting a node in the cluster");
+
+        assertThat(cluster().size(), equalTo(1));
+        ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("1").execute().actionGet();
+        assertThat(healthResponse.isTimedOut(), equalTo(false));
+
+        String indexName = "test";
+
+        logger.info("Creating an index: {} with 2 shards", indexName);
+        createIndex(
+            indexName,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+
+        ensureGreen(indexName);
+
+        logger.info("Writing few docs and searching those which will cache items in RequestCache");
+        indexRandom(true, client.prepareIndex(indexName).setSource("k", "hello"));
+        indexRandom(true, client.prepareIndex(indexName).setSource("y", "hello again"));
+        SearchResponse resp = client.prepareSearch(indexName).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        assertSearchResponse(resp);
+        resp = client.prepareSearch(indexName).setRequestCache(true).setQuery(QueryBuilders.termQuery("y", "hello")).get();
+
+        RequestCacheStats stats = getNodeCacheStats(client);
+        assertTrue(stats.getMemorySizeInBytes() > 0);
+
+        logger.info("Disabling allocation");
+        Settings newSettings = Settings.builder()
+            .put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE.name())
+            .build();
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(newSettings).execute().actionGet();
+
+        logger.info("Starting a second node");
+        String node_2 = internalCluster().startDataOnlyNode(Settings.builder().build());
+        assertThat(cluster().size(), equalTo(2));
+        healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
+        assertThat(healthResponse.isTimedOut(), equalTo(false));
+
+        logger.info("Moving the shard:{} from node:{} to node:{}", indexName + "#0", node_1, node_2);
+        MoveAllocationCommand cmd = new MoveAllocationCommand(indexName, 0, node_1, node_2);
+        internalCluster().client().admin().cluster().prepareReroute().add(cmd).get();
+        ClusterHealthResponse clusterHealth = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final Index index = state.metadata().index(indexName).getIndex();
+
+        assertBusy(() -> {
+            assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(false));
+            assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
+        });
+
+        logger.info("Moving the shard: {} again from node:{} to node:{}", indexName + "#0", node_2, node_1);
+        cmd = new MoveAllocationCommand(indexName, 0, node_2, node_1);
+        internalCluster().client().admin().cluster().prepareReroute().add(cmd).get();
+        clusterHealth = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+
+        assertBusy(() -> {
+            assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+            assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
+        });
+
+        logger.info("Clearing the cache for index:{}. And verify the request stats doesn't go negative", indexName);
+        ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(indexName);
+        client.admin().indices().clearCache(clearIndicesCacheRequest).actionGet();
+
+        stats = getNodeCacheStats(client(node_1));
+        assertTrue(stats.getMemorySizeInBytes() == 0);
+        stats = getNodeCacheStats(client(node_2));
+        assertTrue(stats.getMemorySizeInBytes() == 0);
+    }
+
+    private Path shardDirectory(String server, Index index, int shard) {
+        NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
+        final Path[] paths = env.availableShardPaths(new ShardId(index, shard));
+        assert paths.length == 1;
+        return paths[0];
+    }
+
     private void setupIndex(Client client, String index) throws Exception {
         assertAcked(
             client.admin()
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index 502eae55df83e..c5c49f6dd5dd4 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.index.cache.request;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Accountable;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.metrics.CounterMetric;
@@ -45,13 +47,14 @@
 @PublicApi(since = "1.0.0")
 public final class ShardRequestCache {
 
+    private static final Logger logger = LogManager.getLogger(ShardRequestCache.class);
     final CounterMetric evictionsMetric = new CounterMetric();
     final CounterMetric totalMetric = new CounterMetric();
     final CounterMetric hitCount = new CounterMetric();
     final CounterMetric missCount = new CounterMetric();
 
     public RequestCacheStats stats() {
-        return new RequestCacheStats(totalMetric.count(), evictionsMetric.count(), hitCount.count(), missCount.count());
+        return new RequestCacheStats(Math.max(0, totalMetric.count()), evictionsMetric.count(), hitCount.count(), missCount.count());
     }
 
     public void onHit() {
@@ -76,6 +79,15 @@ public void onRemoval(long keyRamBytesUsed, BytesReference value, boolean evicte
             dec += value.ramBytesUsed();
         }
         totalMetric.dec(dec);
+        if (totalMetric.count() < 0) {
+            totalMetric.inc(dec);
+            logger.warn(
+                "Ignoring the operation to deduct memory: {} from RequestStats memory_size metric as it will "
+                    + "go negative. Current memory: {}. This is a bug.",
+                dec,
+                totalMetric.count()
+            );
+        }
     }
 
     // Old functions which increment size by passing in an Accountable. Functional but no longer used.
@@ -84,15 +96,6 @@ public void onCached(Accountable key, BytesReference value) {
     }
 
     public void onRemoval(Accountable key, BytesReference value, boolean evicted) {
-        if (evicted) {
-            evictionsMetric.inc();
-        }
-        long dec = 0;
-        if (key != null) {
-            dec += key.ramBytesUsed();
-        }
-        if (value != null) {
-            dec += value.ramBytesUsed();
-        }
+        onRemoval(key.ramBytesUsed(), value, evicted);
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 35826d45f969f..1b1820836b66f 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -205,6 +205,11 @@ public final class IndicesRequestCache implements RemovalListener<ICacheKey<Indi
         );
     }
 
+    // package private for testing
+    void invalidateAll() {
+        cache.invalidateAll();
+    }
+
     @Override
     public void close() throws IOException {
         cache.invalidateAll();
@@ -233,8 +238,17 @@ public void onRemoval(RemovalNotification<ICacheKey<Key>, BytesReference> notifi
         // shards as part of request cache.
         // Pass a new removal notification containing Key rather than ICacheKey<Key> to the CacheEntity for backwards compatibility.
         Key key = notification.getKey().key;
-        cacheEntityLookup.apply(key.shardId).ifPresent(entity -> entity.onRemoval(notification));
-        CleanupKey cleanupKey = new CleanupKey(cacheEntityLookup.apply(key.shardId).orElse(null), key.readerCacheKeyId);
+        IndicesService.IndexShardCacheEntity indexShardCacheEntity = (IndicesService.IndexShardCacheEntity) cacheEntityLookup.apply(
+            key.shardId
+        ).orElse(null);
+        if (indexShardCacheEntity != null) {
+            // Here we match the hashcode to avoid scenario where we deduct stats of older IndexShard(with same
+            // shardId) from current IndexShard.
+            if (key.indexShardHashCode == System.identityHashCode(indexShardCacheEntity.getCacheIdentity())) {
+                indexShardCacheEntity.onRemoval(notification);
+            }
+        }
+        CleanupKey cleanupKey = new CleanupKey(indexShardCacheEntity, key.readerCacheKeyId);
         cacheCleanupManager.updateStaleCountOnEntryRemoval(cleanupKey, notification);
     }
 
@@ -266,7 +280,8 @@ BytesReference getOrCompute(
             .getReaderCacheHelper();
         String readerCacheKeyId = delegatingCacheHelper.getDelegatingCacheKey().getId();
         assert readerCacheKeyId != null;
-        final Key key = new Key(((IndexShard) cacheEntity.getCacheIdentity()).shardId(), cacheKey, readerCacheKeyId);
+        IndexShard indexShard = ((IndexShard) cacheEntity.getCacheIdentity());
+        final Key key = new Key(indexShard.shardId(), cacheKey, readerCacheKeyId, System.identityHashCode(indexShard));
         Loader cacheLoader = new Loader(cacheEntity, loader);
         BytesReference value = cache.computeIfAbsent(getICacheKey(key), cacheLoader);
         if (cacheLoader.isLoaded()) {
@@ -299,7 +314,8 @@ void invalidate(IndicesService.IndexShardCacheEntity cacheEntity, DirectoryReade
             IndexReader.CacheHelper cacheHelper = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper();
             readerCacheKeyId = ((OpenSearchDirectoryReader.DelegatingCacheHelper) cacheHelper).getDelegatingCacheKey().getId();
         }
-        cache.invalidate(getICacheKey(new Key(((IndexShard) cacheEntity.getCacheIdentity()).shardId(), cacheKey, readerCacheKeyId)));
+        IndexShard indexShard = (IndexShard) cacheEntity.getCacheIdentity();
+        cache.invalidate(getICacheKey(new Key(indexShard.shardId(), cacheKey, readerCacheKeyId, System.identityHashCode(indexShard))));
     }
 
     /**
@@ -377,19 +393,24 @@ interface CacheEntity extends Accountable {
      */
     static class Key implements Accountable, Writeable {
         public final ShardId shardId; // use as identity equality
+        public final int indexShardHashCode; // While ShardId is usually sufficient to uniquely identify an
+        // indexShard but in case where the same indexShard is deleted and reallocated on same node, we need the
+        // hashcode(default) to identify the older indexShard but with same shardId.
         public final String readerCacheKeyId;
         public final BytesReference value;
 
-        Key(ShardId shardId, BytesReference value, String readerCacheKeyId) {
+        Key(ShardId shardId, BytesReference value, String readerCacheKeyId, int indexShardHashCode) {
             this.shardId = shardId;
             this.value = value;
             this.readerCacheKeyId = Objects.requireNonNull(readerCacheKeyId);
+            this.indexShardHashCode = indexShardHashCode;
         }
 
         Key(StreamInput in) throws IOException {
             this.shardId = in.readOptionalWriteable(ShardId::new);
             this.readerCacheKeyId = in.readOptionalString();
             this.value = in.readBytesReference();
+            this.indexShardHashCode = in.readInt();
         }
 
         @Override
@@ -411,6 +432,7 @@ public boolean equals(Object o) {
             if (!Objects.equals(readerCacheKeyId, key.readerCacheKeyId)) return false;
             if (!shardId.equals(key.shardId)) return false;
             if (!value.equals(key.value)) return false;
+            if (indexShardHashCode != key.indexShardHashCode) return false;
             return true;
         }
 
@@ -419,6 +441,7 @@ public int hashCode() {
             int result = shardId.hashCode();
             result = 31 * result + readerCacheKeyId.hashCode();
             result = 31 * result + value.hashCode();
+            result = 31 * result + indexShardHashCode;
             return result;
         }
 
@@ -427,6 +450,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(shardId);
             out.writeOptionalString(readerCacheKeyId);
             out.writeBytesReference(value);
+            out.writeInt(indexShardHashCode);
         }
     }
 
diff --git a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
index af657dadd7a1a..a5014675ce0ed 100644
--- a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
+++ b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
@@ -45,6 +45,7 @@ private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random,
             value[i] = (byte) (random.nextInt(126 - 32) + 32);
         }
         BytesReference keyValue = new BytesArray(value);
-        return new IndicesRequestCache.Key(shard, keyValue, UUID.randomUUID().toString()); // same UUID source as used in real key
+        return new IndicesRequestCache.Key(shard, keyValue, UUID.randomUUID().toString(), shard.hashCode()); // same UUID
+        // source as used in real key
     }
 }
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index bbf2867a0087c..bc99c895cb782 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -82,14 +82,21 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.opensearch.indices.IndicesRequestCache.INDEX_DIMENSION_NAME;
+import static org.opensearch.indices.IndicesRequestCache.INDICES_CACHE_QUERY_SIZE;
 import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
 import static org.opensearch.indices.IndicesRequestCache.SHARD_ID_DIMENSION_NAME;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -460,7 +467,12 @@ public void testStaleCount_OnRemovalNotificationOfStaleKey_DecrementsStaleCount(
         // cache count should not be affected
         assertEquals(2, cache.count());
 
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(reader));
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(reader),
+            indexShard.hashCode()
+        );
         // test the mapping
         ConcurrentMap<ShardId, HashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
         // shard id should exist
@@ -517,7 +529,12 @@ public void testStaleCount_OnRemovalNotificationOfNonStaleKey_DoesNotDecrementsS
         assertEquals(2, cache.count());
 
         // evict entry from second reader (this reader is not closed)
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader));
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(secondReader),
+            indexShard.hashCode()
+        );
 
         // test the mapping
         ConcurrentMap<ShardId, HashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
@@ -567,7 +584,12 @@ public void testStaleCount_WithoutReaderClosing_DecrementsStaleCount() throws Ex
         // no keys are stale
         assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
         // create notification for removal of non-stale entry
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(reader));
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(reader),
+            indexShard.hashCode()
+        );
         cache.onRemoval(
             new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
                 new ICacheKey<>(key),
@@ -610,11 +632,8 @@ public void testStaleCount_OnRemovalNotifications() throws Exception {
         assertEquals(totalKeys, cache.cacheCleanupManager.getStaleKeysCount().get());
 
         String readerCacheKeyId = getReaderCacheKeyId(reader);
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
-            ((IndexShard) entity.getCacheIdentity()).shardId(),
-            termBytes,
-            readerCacheKeyId
-        );
+        IndexShard indexShard = (IndexShard) entity.getCacheIdentity();
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(indexShard.shardId(), termBytes, readerCacheKeyId, indexShard.hashCode());
 
         int staleCount = cache.cacheCleanupManager.getStaleKeysCount().get();
         // Notification for Replaced should not deduct the staleCount
@@ -709,7 +728,12 @@ public void testCleanupKeyToCountMapAreSetAppropriately() throws Exception {
         // second reader's mapping should not be affected
         assertEquals(2, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
         // send removal notification for first reader
-        IndicesRequestCache.Key key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(reader));
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(reader),
+            indexShard.hashCode()
+        );
         cache.onRemoval(
             new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
                 new ICacheKey<>(key),
@@ -725,7 +749,7 @@ public void testCleanupKeyToCountMapAreSetAppropriately() throws Exception {
         assertEquals(2, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
 
         // Without closing the secondReader send removal notification of one of its key
-        key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader));
+        key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader), indexShard.hashCode());
         cache.onRemoval(
             new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
                 new ICacheKey<>(key),
@@ -738,7 +762,7 @@ public void testCleanupKeyToCountMapAreSetAppropriately() throws Exception {
         // secondReader's readerCacheKeyId count should be decremented by 1
         assertEquals(1, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
         // Without closing the secondReader send removal notification of its last key
-        key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader));
+        key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader), indexShard.hashCode());
         cache.onRemoval(
             new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
                 new ICacheKey<>(key),
@@ -1152,11 +1176,11 @@ public void testEqualsKey() throws IOException {
         IOUtils.close(reader1, reader2, writer, dir);
         IndexShard indexShard = mock(IndexShard.class);
         when(indexShard.state()).thenReturn(IndexShardState.STARTED);
-        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1);
-        IndicesRequestCache.Key key2 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1);
-        IndicesRequestCache.Key key3 = new IndicesRequestCache.Key(shardId1, new TestBytesReference(1), rKey1);
-        IndicesRequestCache.Key key4 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey2);
-        IndicesRequestCache.Key key5 = new IndicesRequestCache.Key(shardId, new TestBytesReference(2), rKey2);
+        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1, shardId.hashCode());
+        IndicesRequestCache.Key key2 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1, shardId.hashCode());
+        IndicesRequestCache.Key key3 = new IndicesRequestCache.Key(shardId1, new TestBytesReference(1), rKey1, shardId1.hashCode());
+        IndicesRequestCache.Key key4 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey2, shardId.hashCode());
+        IndicesRequestCache.Key key5 = new IndicesRequestCache.Key(shardId, new TestBytesReference(2), rKey2, shardId.hashCode());
         String s = "Some other random object";
         assertEquals(key1, key1);
         assertEquals(key1, key2);
@@ -1170,7 +1194,12 @@ public void testEqualsKey() throws IOException {
     public void testSerializationDeserializationOfCacheKey() throws Exception {
         IndicesService.IndexShardCacheEntity shardCacheEntity = new IndicesService.IndexShardCacheEntity(indexShard);
         String readerCacheKeyId = UUID.randomUUID().toString();
-        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), readerCacheKeyId);
+        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            readerCacheKeyId,
+            indexShard.hashCode()
+        );
         BytesReference bytesReference = null;
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             key1.writeTo(out);
@@ -1183,6 +1212,100 @@ public void testSerializationDeserializationOfCacheKey() throws Exception {
         assertEquals(readerCacheKeyId, key2.readerCacheKeyId);
         assertEquals(((IndexShard) shardCacheEntity.getCacheIdentity()).shardId(), key2.shardId);
         assertEquals(getTermBytes(), key2.value);
+        assertEquals(indexShard.hashCode(), key2.indexShardHashCode);
+    }
+
+    public void testGetOrComputeConcurrentlyWithMultipleIndices() throws Exception {
+        threadPool = getThreadPool();
+        int numberOfIndices = randomIntBetween(2, 5);
+        List<String> indicesList = new ArrayList<>();
+        List<IndexShard> indexShardList = Collections.synchronizedList(new ArrayList<>());
+        for (int i = 0; i < numberOfIndices; i++) {
+            String indexName = "test" + i;
+            indicesList.add(indexName);
+            IndexShard indexShard = createIndex(
+                indexName,
+                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+            ).getShard(0);
+            indexShardList.add(indexShard);
+        }
+        // Create a cache with 2kb to cause evictions and test that flow as well.
+        IndicesRequestCache cache = getIndicesRequestCache(Settings.builder().put(INDICES_CACHE_QUERY_SIZE.getKey(), "2kb").build());
+        Map<IndexShard, DirectoryReader> readerMap = new ConcurrentHashMap<>();
+        Map<IndexShard, IndicesService.IndexShardCacheEntity> entityMap = new ConcurrentHashMap<>();
+        Map<IndexShard, IndexWriter> writerMap = new ConcurrentHashMap<>();
+        int numberOfItems = randomIntBetween(200, 400);
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            entityMap.put(indexShard, new IndicesService.IndexShardCacheEntity(indexShard));
+            Directory dir = newDirectory();
+            IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+            for (int j = 0; j < numberOfItems; j++) {
+                writer.addDocument(newDoc(j, generateString(randomIntBetween(4, 50))));
+            }
+            writerMap.put(indexShard, writer);
+            DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+            readerMap.put(indexShard, reader);
+        }
+
+        CountDownLatch latch = new CountDownLatch(numberOfItems);
+        ExecutorService executorService = Executors.newFixedThreadPool(5);
+        for (int i = 0; i < numberOfItems; i++) {
+            int finalI = i;
+            executorService.submit(() -> {
+                int randomIndexPosition = randomIntBetween(0, numberOfIndices - 1);
+                IndexShard indexShard = indexShardList.get(randomIndexPosition);
+                TermQueryBuilder termQuery = new TermQueryBuilder("id", generateString(randomIntBetween(4, 50)));
+                BytesReference termBytes = null;
+                try {
+                    termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                Loader loader = new Loader(readerMap.get(indexShard), finalI);
+                try {
+                    cache.getOrCompute(entityMap.get(indexShard), loader, readerMap.get(indexShard), termBytes);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+                latch.countDown();
+            });
+        }
+        latch.await();
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            IndicesService.IndexShardCacheEntity entity = entityMap.get(indexShard);
+            RequestCacheStats stats = entity.stats().stats();
+            assertTrue(stats.getMemorySizeInBytes() >= 0);
+            assertTrue(stats.getMissCount() >= 0);
+            assertTrue(stats.getEvictions() >= 0);
+        }
+        cache.invalidateAll();
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            IndicesService.IndexShardCacheEntity entity = entityMap.get(indexShard);
+            RequestCacheStats stats = entity.stats().stats();
+            assertEquals(0, stats.getMemorySizeInBytes());
+        }
+
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            readerMap.get(indexShard).close();
+            writerMap.get(indexShard).close();
+            writerMap.get(indexShard).getDirectory().close();
+        }
+        IOUtils.close(cache);
+        executorService.shutdownNow();
+    }
+
+    public static String generateString(int length) {
+        String characters = "abcdefghijklmnopqrstuvwxyz";
+        StringBuilder sb = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            int index = randomInt(characters.length() - 1);
+            sb.append(characters.charAt(index));
+        }
+        return sb.toString();
     }
 
     private class TestBytesReference extends AbstractBytesReference {

From 95dcce8e633b5a67f366f109560751ac731e90fb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 22:15:40 -0400
Subject: [PATCH 21/52] Bump org.hdrhistogram:HdrHistogram from 2.1.12 to 2.2.1
 in /server (#13556)

* Bump org.hdrhistogram:HdrHistogram from 2.1.12 to 2.2.1 in /server

Bumps [org.hdrhistogram:HdrHistogram](https://github.com/HdrHistogram/HdrHistogram) from 2.1.12 to 2.2.1.
- [Release notes](https://github.com/HdrHistogram/HdrHistogram/releases)
- [Commits](https://github.com/HdrHistogram/HdrHistogram/compare/HdrHistogram-2.1.12...HdrHistogram-2.2.1)

---
updated-dependencies:
- dependency-name: org.hdrhistogram:HdrHistogram
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                 | 1 +
 server/build.gradle                          | 2 +-
 server/licenses/HdrHistogram-2.1.12.jar.sha1 | 1 -
 server/licenses/HdrHistogram-2.2.1.jar.sha1  | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 server/licenses/HdrHistogram-2.1.12.jar.sha1
 create mode 100644 server/licenses/HdrHistogram-2.2.1.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 43975a79e9cd7..8215c2983724a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `lycheeverse/lychee-action` from 1.9.3 to 1.10.0 ([#13447](https://github.com/opensearch-project/OpenSearch/pull/13447))
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
 - Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
+- Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
 
 ### Changed
 - [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
diff --git a/server/build.gradle b/server/build.gradle
index fc383f940991c..9714f13ec67d6 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -98,7 +98,7 @@ dependencies {
   // percentiles aggregation
   api 'com.tdunning:t-digest:3.3'
   // precentil ranks aggregation
-  api 'org.hdrhistogram:HdrHistogram:2.1.12'
+  api 'org.hdrhistogram:HdrHistogram:2.2.1'
 
   // lucene spatial
   api "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}", optional
diff --git a/server/licenses/HdrHistogram-2.1.12.jar.sha1 b/server/licenses/HdrHistogram-2.1.12.jar.sha1
deleted file mode 100644
index 9d20fa0e5f22d..0000000000000
--- a/server/licenses/HdrHistogram-2.1.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6eb7552156e0d517ae80cc2247be1427c8d90452
\ No newline at end of file
diff --git a/server/licenses/HdrHistogram-2.2.1.jar.sha1 b/server/licenses/HdrHistogram-2.2.1.jar.sha1
new file mode 100644
index 0000000000000..68225950d4744
--- /dev/null
+++ b/server/licenses/HdrHistogram-2.2.1.jar.sha1
@@ -0,0 +1 @@
+0eb1feb351f64176c377772a30174e582c0274d5
\ No newline at end of file

From a228dfc824772bb1e195233d5c734fde3ad03937 Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Tue, 7 May 2024 14:48:50 +0800
Subject: [PATCH 22/52] Update supported version for the bug fix of from and
 size parameter can be negative when searching (#13408)

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 .../rest-api-spec/test/search/360_from_and_size.yml       | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml
index 95bcb9e5326cb..7f3fb77b86366 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml
@@ -37,8 +37,8 @@ teardown:
 ---
 "Throws exception if from or size query parameter is negative":
   - skip:
-      version: " - 2.99.99"
-      reason: "fixed in 3.0.0"
+      version: " - 2.13.99"
+      reason: "fixed in 2.14.0"
   - do:
       catch: '/\[from\] parameter cannot be negative, found \[-5\]/'
       search:
@@ -76,8 +76,8 @@ teardown:
 ---
 "Throws exception if from or size request body parameter is negative":
   - skip:
-      version: " - 2.99.99"
-      reason: "fixed in 3.0.0"
+      version: " - 2.13.99"
+      reason: "fixed in 2.14.0"
   - do:
       catch: '/\[from\] parameter cannot be negative, found \[-5\]/'
       search:

From 6b6a0a2b3c3fc278b2af0cacfbfe7dce0f6bc7e7 Mon Sep 17 00:00:00 2001
From: Varun Bansal <bansvaru@amazon.com>
Date: Tue, 7 May 2024 18:45:23 +0530
Subject: [PATCH 23/52] add TestIssueLogging on flaky test -
 org.opensearch.remotestore.RemoteStoreRestoreIT.testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown
 (#13489)

* add TestIssueLogging on flaky test - org.opensearch.remotestore.RemoteStoreRestoreIT.testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown

Signed-off-by: Varun Bansal <bansvaru@amazon.com>
---
 .../java/org/opensearch/remotestore/RemoteStoreRestoreIT.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
index 195b2d18df7cc..bd84ab026dcea 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
@@ -27,6 +27,7 @@
 import org.opensearch.repositories.fs.ReloadableFsRepository;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.junit.annotations.TestIssueLogging;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -91,6 +92,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws Except
      * Simulates all data restored using Remote Translog Store.
      * @throws IOException IO Exception.
      */
+    @TestIssueLogging(value = "_root:TRACE", issueUrl = "https://github.com/opensearch-project/OpenSearch/issues/11085")
     public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws Exception {
         testRestoreFlowBothPrimaryReplicasDown(1, false, true, randomIntBetween(1, 5));
     }

From 4eb33b0720abe67d644d8e8abfff4667ecd531fe Mon Sep 17 00:00:00 2001
From: Andrew Ross <andrross@amazon.com>
Date: Tue, 7 May 2024 13:40:09 -0700
Subject: [PATCH 24/52] Forward port 2.14 release notes to main (#13589)

Signed-off-by: Andrew Ross <andrross@amazon.com>
---
 CHANGELOG.md                                  | 77 ----------------
 .../opensearch.release-notes-2.14.0.md        | 87 +++++++++++++++++++
 2 files changed, 87 insertions(+), 77 deletions(-)
 create mode 100644 release-notes/opensearch.release-notes-2.14.0.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8215c2983724a..db44887a0e59f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,79 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
-- Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285))
-- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
-- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
-- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
-- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
-- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
-- Rename ingest processor supports overriding target field if exists ([#12990](https://github.com/opensearch-project/OpenSearch/pull/12990))
-- [Tiered Caching] Make took time caching policy setting dynamic ([#13063](https://github.com/opensearch-project/OpenSearch/pull/13063))
-- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
-- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
-- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))
-- [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
-- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
-- Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
-- [Tiered Caching] Add dimension-based stats to ICache implementations. ([#12531](https://github.com/opensearch-project/OpenSearch/pull/12531))
-- Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
-- Add an individual setting of rate limiter for segment replication ([#12959](https://github.com/opensearch-project/OpenSearch/pull/12959))
-- [Tiered Caching] Add dimension-based stats to TieredSpilloverCache ([#13236](https://github.com/opensearch-project/OpenSearch/pull/13236))
-- [Tiered Caching] Expose new cache stats API ([#13237](https://github.com/opensearch-project/OpenSearch/pull/13237))
-- [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
-- Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
-- Support wildcard/regex for indices param in _remotestore/_restore ([#8922](https://github.com/opensearch-project/OpenSearch/pull/8922))
-- [Tiered Caching] Gate new stats logic behind FeatureFlags.PLUGGABLE_CACHE ([#13238](https://github.com/opensearch-project/OpenSearch/pull/13238))
-- [Tiered Caching] Add a dynamic setting to disable/enable disk cache. ([#13373](https://github.com/opensearch-project/OpenSearch/pull/13373))
-- [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
-- [Batch Ingestion] Add `batch_size` to `_bulk` API. ([#12457](https://github.com/opensearch-project/OpenSearch/issues/12457))
-- [Tiered caching] Make Indices Request Cache Stale Key Mgmt Threshold setting dynamic ([#12941](https://github.com/opensearch-project/OpenSearch/pull/12941))
-- Batch mode for async fetching shard information in GatewayAllocator for unassigned shards ([#8746](https://github.com/opensearch-project/OpenSearch/pull/8746))
-- [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225))
-- [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))
-- [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276))
-- Add support for deep copying SearchRequest ([#12295](https://github.com/opensearch-project/OpenSearch/pull/12295))
-- Support multi ranges traversal when doing date histogram rewrite optimization. ([#13317](https://github.com/opensearch-project/OpenSearch/pull/13317))
 
 ### Dependencies
-- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
-- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
-- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893), [#13117](https://github.com/opensearch-project/OpenSearch/pull/13117))
-- Bump `netty` from 4.1.107.Final to 4.1.109.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924), [#13233](https://github.com/opensearch-project/OpenSearch/pull/13233))
-- Bump `commons-io:commons-io` from 2.15.1 to 2.16.0 ([#12996](https://github.com/opensearch-project/OpenSearch/pull/12996), [#12998](https://github.com/opensearch-project/OpenSearch/pull/12998), [#12999](https://github.com/opensearch-project/OpenSearch/pull/12999))
-- Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.26.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
-- Bump `org.apache.commons:commonscodec` from 1.15 to 1.16.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
-- Bump `org.apache.commons:commonslang` from 3.13.0 to 3.14.0 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
-- Bump Apache Tika from 2.6.0 to 2.9.2 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
-- Bump `com.gradle.enterprise` from 3.16.2 to 3.17.2 ([#13116](https://github.com/opensearch-project/OpenSearch/pull/13116), [#13191](https://github.com/opensearch-project/OpenSearch/pull/13191), [#13327](https://github.com/opensearch-project/OpenSearch/pull/13327))
-- Bump `gradle/wrapper-validation-action` from 2 to 3 ([#13192](https://github.com/opensearch-project/OpenSearch/pull/13192))
-- Bump joda from 2.12.2 to 2.12.7 ([#13193](https://github.com/opensearch-project/OpenSearch/pull/13193))
-- Bump bouncycastle from 1.77 to 1.78 ([#13243](https://github.com/opensearch-project/OpenSearch/pull/13243))
-- Update google dependencies in repository-gcs and discovery-gce ([#13213](https://github.com/opensearch-project/OpenSearch/pull/13213))
-- Bump `com.google.apis:google-api-services-compute` from v1-rev235-1.25.0 to v1-rev20240407-2.0.0 ([#13333](https://github.com/opensearch-project/OpenSearch/pull/13333))
-- Bump `commons-cli:commons-cli` from 1.6.0 to 1.7.0 ([#13331](https://github.com/opensearch-project/OpenSearch/pull/13331))
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
-- Bump `jakarta.enterprise:jakarta.enterprise.cdi-api` from 4.0.1 to 4.1.0 ([#13328](https://github.com/opensearch-project/OpenSearch/pull/13328))
-- Bump `com.google.api.grpc:proto-google-iam-v1` from 0.12.0 to 1.33.0 ([#13332](https://github.com/opensearch-project/OpenSearch/pull/13332))
-- OpenJDK Update (April 2024 Patch releases), update to Eclipse Temurin 21.0.3+9 ([#13389](https://github.com/opensearch-project/OpenSearch/pull/13389))
-- Bump `com.squareup.okio:okio` from 3.8.0 to 3.9.0 ([#12997](https://github.com/opensearch-project/OpenSearch/pull/12997))
-- Bump `com.netflix.nebula.ospackage-base` from 11.8.1 to 11.9.0 ([#13440](https://github.com/opensearch-project/OpenSearch/pull/13440))
-- Bump `org.bouncycastle:bc-fips` from 1.0.2.4 to 1.0.2.5 ([#13446](https://github.com/opensearch-project/OpenSearch/pull/13446))
-- Bump `lycheeverse/lychee-action` from 1.9.3 to 1.10.0 ([#13447](https://github.com/opensearch-project/OpenSearch/pull/13447))
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
 - Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
 
 ### Changed
-- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
-- Improve built-in secure transports support ([#12907](https://github.com/opensearch-project/OpenSearch/pull/12907))
-- Update links to documentation in rest-api-spec ([#13043](https://github.com/opensearch-project/OpenSearch/pull/13043))
-- Ignoring unavailable shards during search request execution with ignore_available parameter ([#13298](https://github.com/opensearch-project/OpenSearch/pull/13298))
-- Refactoring globMatch using simpleMatchWithNormalizedStrings from Regex ([#13104](https://github.com/opensearch-project/OpenSearch/pull/13104))
-- [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292))
-- Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412))
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
-- [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
 - Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
 
 ### Deprecated
@@ -87,19 +23,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove handling of index.mapper.dynamic in AutoCreateIndex([#13067](https://github.com/opensearch-project/OpenSearch/pull/13067))
 
 ### Fixed
-- Fix bulk API ignores ingest pipeline for upsert ([#12883](https://github.com/opensearch-project/OpenSearch/pull/12883))
-- Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
-- Fix UOE While building Exists query for nested search_as_you_type field ([#12048](https://github.com/opensearch-project/OpenSearch/pull/12048))
-- Client with Java 8 runtime and Apache HttpClient 5 Transport fails with java.lang.NoSuchMethodError: java.nio.ByteBuffer.flip()Ljava/nio/ByteBuffer ([#13100](https://github.com/opensearch-project/opensearch-java/pull/13100))
-- Fix from and size parameter can be negative when searching ([#13047](https://github.com/opensearch-project/OpenSearch/pull/13047))
-- Enabled mockTelemetryPlugin for IT and fixed OOM issues ([#13054](https://github.com/opensearch-project/OpenSearch/pull/13054))
-- Fix implement mark() and markSupported() in class FilterStreamInput ([#13098](https://github.com/opensearch-project/OpenSearch/pull/13098))
-- Fix IndicesRequestCache Stale calculation ([#13070](https://github.com/opensearch-project/OpenSearch/pull/13070)]
-- Fix snapshot _status API to return correct status for partial snapshots ([#12812](https://github.com/opensearch-project/OpenSearch/pull/12812))
-- Improve the error messages for _stats with closed indices ([#13012](https://github.com/opensearch-project/OpenSearch/pull/13012))
-- Ignore BaseRestHandler unconsumed content check as it's always consumed. ([#13290](https://github.com/opensearch-project/OpenSearch/pull/13290))
-- Fix mapper_parsing_exception when using flat_object fields with names longer than 11 characters ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13259))
-- DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
 - Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 
 ### Security
diff --git a/release-notes/opensearch.release-notes-2.14.0.md b/release-notes/opensearch.release-notes-2.14.0.md
new file mode 100644
index 0000000000000..8ef0215baa67a
--- /dev/null
+++ b/release-notes/opensearch.release-notes-2.14.0.md
@@ -0,0 +1,87 @@
+## 2024-05-07 Version 2.14.0 Release Notes
+
+## [2.14.0]
+### Added
+- Add explicit dependency to validatePom and generatePom tasks ([#12909](https://github.com/opensearch-project/OpenSearch/pull/12909))
+- Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285))
+- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
+- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
+- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
+- Add a counter to node stat (and _cat/shards) api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
+- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
+- Rename ingest processor supports overriding target field if exists ([#12990](https://github.com/opensearch-project/OpenSearch/pull/12990))
+- [Tiered Caching] Make took time caching policy setting dynamic ([#13063](https://github.com/opensearch-project/OpenSearch/pull/13063))
+- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
+- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))
+- [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
+- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
+- Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
+- [Tiered Caching] Add dimension-based stats to ICache implementations. ([#12531](https://github.com/opensearch-project/OpenSearch/pull/12531))
+- Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
+- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
+- Add an individual setting of rate limiter for segment replication ([#12959](https://github.com/opensearch-project/OpenSearch/pull/12959))
+- [Tiered Caching] Add dimension-based stats to TieredSpilloverCache ([#13236](https://github.com/opensearch-project/OpenSearch/pull/13236))
+- [Tiered Caching] Expose new cache stats API ([#13237](https://github.com/opensearch-project/OpenSearch/pull/13237))
+- [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
+- Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
+- [Tiered caching] Make Indices Request Cache Stale Key Mgmt Threshold setting dynamic ([#12941](https://github.com/opensearch-project/OpenSearch/pull/12941))
+- Make search query counters dynamic to support all query types ([#12601](https://github.com/opensearch-project/OpenSearch/pull/12601))
+- [Tiered Caching] Gate new stats logic behind FeatureFlags.PLUGGABLE_CACHE ([#13238](https://github.com/opensearch-project/OpenSearch/pull/13238))
+- [Tiered Caching] Add a dynamic setting to disable/enable disk cache. ([#13373](https://github.com/opensearch-project/OpenSearch/pull/13373))
+- Batch mode for async fetching shard information in GatewayAllocator for unassigned shards ([#8746](https://github.com/opensearch-project/OpenSearch/pull/8746))
+- [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225))
+- [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))
+- [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276))
+- [Batch Ingestion] Add `batch_size` to `_bulk` API. ([#12457](https://github.com/opensearch-project/OpenSearch/issues/12457))
+- [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
+- Support multi ranges traversal when doing date histogram rewrite optimization. ([#13317](https://github.com/opensearch-project/OpenSearch/pull/13317))
+
+### Dependencies
+- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
+- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
+- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893), [#13117](https://github.com/opensearch-project/OpenSearch/pull/13117))
+- Bump `netty` from 4.1.107.Final to 4.1.109.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924), [#13233](https://github.com/opensearch-project/OpenSearch/pull/13233))
+- Bump `commons-io:commons-io` from 2.15.1 to 2.16.0 ([#12996](https://github.com/opensearch-project/OpenSearch/pull/12996), [#12998](https://github.com/opensearch-project/OpenSearch/pull/12998), [#12999](https://github.com/opensearch-project/OpenSearch/pull/12999))
+- Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.26.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump `org.apache.commons:commonscodec` from 1.15 to 1.16.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump `org.apache.commons:commonslang` from 3.13.0 to 3.14.0 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump Apache Tika from 2.6.0 to 2.9.2 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump `com.gradle.enterprise` from 3.16.2 to 3.17.2 ([#13116](https://github.com/opensearch-project/OpenSearch/pull/13116), [#13191](https://github.com/opensearch-project/OpenSearch/pull/13191), [#13327](https://github.com/opensearch-project/OpenSearch/pull/13327))
+- Bump `gradle/wrapper-validation-action` from 2 to 3 ([#13192](https://github.com/opensearch-project/OpenSearch/pull/13192))
+- Bump joda from 2.12.2 to 2.12.7 ([#13193](https://github.com/opensearch-project/OpenSearch/pull/13193))
+- Bump bouncycastle from 1.77 to 1.78 ([#13243](https://github.com/opensearch-project/OpenSearch/pull/13243))
+- Update google dependencies in repository-gcs and discovery-gce ([#13213](https://github.com/opensearch-project/OpenSearch/pull/13213))
+- Bump `com.google.apis:google-api-services-compute` from v1-rev235-1.25.0 to v1-rev20240407-2.0.0 ([#13333](https://github.com/opensearch-project/OpenSearch/pull/13333))
+- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.11 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329))
+- Bump `commons-cli:commons-cli` from 1.6.0 to 1.7.0 ([#13331](https://github.com/opensearch-project/OpenSearch/pull/13331))
+- Bump `jakarta.enterprise:jakarta.enterprise.cdi-api` from 4.0.1 to 4.1.0 ([#13328](https://github.com/opensearch-project/OpenSearch/pull/13328))
+- Bump `com.google.api.grpc:proto-google-iam-v1` from 0.12.0 to 1.33.0 ([#13332](https://github.com/opensearch-project/OpenSearch/pull/13332))
+- Bump `com.squareup.okio:okio` from 3.8.0 to 3.9.0 ([#12997](https://github.com/opensearch-project/OpenSearch/pull/12997))
+- OpenJDK Update (April 2024 Patch releases), update to Eclipse Temurin 21.0.3+9 ([#13389](https://github.com/opensearch-project/OpenSearch/pull/13389))
+- Bump `org.bouncycastle:bc-fips` from 1.0.2.4 to 1.0.2.5 ([#13446](https://github.com/opensearch-project/OpenSearch/pull/13446))
+- Bump `lycheeverse/lychee-action` from 1.9.3 to 1.10.0 ([#13447](https://github.com/opensearch-project/OpenSearch/pull/13447))
+- Bump `com.netflix.nebula.ospackage-base` from 11.8.1 to 11.9.0 ([#13440](https://github.com/opensearch-project/OpenSearch/pull/13440))
+
+### Changed
+- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
+- Improve built-in secure transports support ([#12907](https://github.com/opensearch-project/OpenSearch/pull/12907))
+- Update links to documentation in rest-api-spec ([#13043](https://github.com/opensearch-project/OpenSearch/pull/13043))
+- Ignoring unavailable shards during search request execution with ignore_available parameter ([#13298](https://github.com/opensearch-project/OpenSearch/pull/13298))
+- Refactoring globMatch using simpleMatchWithNormalizedStrings from Regex ([#13104](https://github.com/opensearch-project/OpenSearch/pull/13104))
+- [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292))
+- Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412))
+- [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
+
+### Fixed
+- Fix bulk API ignores ingest pipeline for upsert ([#12883](https://github.com/opensearch-project/OpenSearch/pull/12883))
+- Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
+- Fix UOE While building Exists query for nested search_as_you_type field ([#12048](https://github.com/opensearch-project/OpenSearch/pull/12048))
+- Enabled mockTelemetryPlugin for IT and fixed OOM issues ([#13054](https://github.com/opensearch-project/OpenSearch/pull/13054))
+- Fix from and size parameter can be negative when searching ([#13047](https://github.com/opensearch-project/OpenSearch/pull/13047))
+- Fix implement mark() and markSupported() in class FilterStreamInput ([#13098](https://github.com/opensearch-project/OpenSearch/pull/13098))
+- Fix IndicesRequestCache Stale calculation ([#13070](https://github.com/opensearch-project/OpenSearch/pull/13070)]
+- Fix snapshot _status API to return correct status for partial snapshots ([#12812](https://github.com/opensearch-project/OpenSearch/pull/12812))
+- Improve the error messages for _stats with closed indices ([#13012](https://github.com/opensearch-project/OpenSearch/pull/13012))
+- Ignore BaseRestHandler unconsumed content check as it's always consumed. ([#13290](https://github.com/opensearch-project/OpenSearch/pull/13290))
+- Fix mapper_parsing_exception when using flat_object fields with names longer than 11 characters ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13259))
+- DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
\ No newline at end of file

From 48da1b8e0b901601c9036fc3eeb34146b6fd22c4 Mon Sep 17 00:00:00 2001
From: Sagar <99425694+sgup432@users.noreply.github.com>
Date: Wed, 8 May 2024 09:15:36 -0700
Subject: [PATCH 25/52] Fix IndicesRequestCache clean up logic (#13597)

Signed-off-by: Sagar Upadhyaya <upasagar@amazon.com>
Co-authored-by: Sagar Upadhyaya <upasagar@amazon.com>
---
 .../indices/IndicesRequestCache.java          |  28 ++--
 .../IRCKeyWriteableSerializerTests.java       |   6 +-
 .../indices/IndicesRequestCacheTests.java     | 121 ++++++++++++++++++
 3 files changed, 144 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 1b1820836b66f..44af83bb839c1 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -53,6 +53,7 @@
 import org.opensearch.common.cache.service.CacheService;
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.config.CacheConfig;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Setting;
@@ -410,7 +411,8 @@ static class Key implements Accountable, Writeable {
             this.shardId = in.readOptionalWriteable(ShardId::new);
             this.readerCacheKeyId = in.readOptionalString();
             this.value = in.readBytesReference();
-            this.indexShardHashCode = in.readInt();
+            this.indexShardHashCode = in.readInt(); // We are serializing/de-serializing this as we need to store the
+            // key as part of tiered/disk cache. The key is not passed between nodes at this point.
         }
 
         @Override
@@ -450,7 +452,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(shardId);
             out.writeOptionalString(readerCacheKeyId);
             out.writeBytesReference(value);
-            out.writeInt(indexShardHashCode);
+            out.writeInt(indexShardHashCode); // We are serializing/de-serializing this as we need to store the
+            // key as part of tiered/disk cache. The key is not passed between nodes at this point.
         }
     }
 
@@ -713,15 +716,16 @@ private synchronized void cleanCache(double stalenessThreshold) {
             // Contains CleanupKey objects with open shard but invalidated readerCacheKeyId.
             final Set<CleanupKey> cleanupKeysFromOutdatedReaders = new HashSet<>();
             // Contains CleanupKey objects of a closed shard.
-            final Set<Object> cleanupKeysFromClosedShards = new HashSet<>();
+            final Set<Tuple<ShardId, Integer>> cleanupKeysFromClosedShards = new HashSet<>();
 
             for (Iterator<CleanupKey> iterator = keysToClean.iterator(); iterator.hasNext();) {
                 CleanupKey cleanupKey = iterator.next();
                 iterator.remove();
                 if (cleanupKey.readerCacheKeyId == null || !cleanupKey.entity.isOpen()) {
                     // null indicates full cleanup, as does a closed shard
-                    ShardId shardId = ((IndexShard) cleanupKey.entity.getCacheIdentity()).shardId();
-                    cleanupKeysFromClosedShards.add(shardId);
+                    IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
+                    // Add both shardId and indexShardHashCode to uniquely identify an indexShard.
+                    cleanupKeysFromClosedShards.add(new Tuple<>(indexShard.shardId(), indexShard.hashCode()));
                 } else {
                     cleanupKeysFromOutdatedReaders.add(cleanupKey);
                 }
@@ -735,14 +739,22 @@ private synchronized void cleanCache(double stalenessThreshold) {
 
             for (Iterator<ICacheKey<Key>> iterator = cache.keys().iterator(); iterator.hasNext();) {
                 ICacheKey<Key> key = iterator.next();
-                if (cleanupKeysFromClosedShards.contains(key.key.shardId)) {
+                Key delegatingKey = key.key;
+                if (cleanupKeysFromClosedShards.contains(new Tuple<>(delegatingKey.shardId, delegatingKey.indexShardHashCode))) {
                     // Since the shard is closed, the cache should drop stats for this shard.
                     dimensionListsToDrop.add(key.dimensions);
                     iterator.remove();
                 } else {
-                    CleanupKey cleanupKey = new CleanupKey(cacheEntityLookup.apply(key.key.shardId).orElse(null), key.key.readerCacheKeyId);
-                    if (cleanupKeysFromOutdatedReaders.contains(cleanupKey)) {
+                    CacheEntity cacheEntity = cacheEntityLookup.apply(delegatingKey.shardId).orElse(null);
+                    if (cacheEntity == null) {
+                        // If cache entity is null, it means that index or shard got deleted/closed meanwhile.
+                        // So we will delete this key.
                         iterator.remove();
+                    } else {
+                        CleanupKey cleanupKey = new CleanupKey(cacheEntity, delegatingKey.readerCacheKeyId);
+                        if (cleanupKeysFromOutdatedReaders.contains(cleanupKey)) {
+                            iterator.remove();
+                        }
                     }
                 }
             }
diff --git a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
index a5014675ce0ed..fb5c0a3f9c8f7 100644
--- a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
+++ b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
@@ -30,7 +30,7 @@ public void testSerializer() throws Exception {
         Random rand = Randomness.get();
         for (int valueLength : valueLengths) {
             for (int i = 0; i < NUM_KEYS; i++) {
-                IndicesRequestCache.Key key = getRandomIRCKey(valueLength, rand, indexShard.shardId());
+                IndicesRequestCache.Key key = getRandomIRCKey(valueLength, rand, indexShard.shardId(), System.identityHashCode(indexShard));
                 byte[] serialized = ser.serialize(key);
                 assertTrue(ser.equals(key, serialized));
                 IndicesRequestCache.Key deserialized = ser.deserialize(serialized);
@@ -39,13 +39,13 @@ public void testSerializer() throws Exception {
         }
     }
 
-    private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random, ShardId shard) {
+    private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random, ShardId shard, int indexShardHashCode) {
         byte[] value = new byte[valueLength];
         for (int i = 0; i < valueLength; i++) {
             value[i] = (byte) (random.nextInt(126 - 32) + 32);
         }
         BytesReference keyValue = new BytesArray(value);
-        return new IndicesRequestCache.Key(shard, keyValue, UUID.randomUUID().toString(), shard.hashCode()); // same UUID
+        return new IndicesRequestCache.Key(shard, keyValue, UUID.randomUUID().toString(), indexShardHashCode); // same UUID
         // source as used in real key
     }
 }
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index bc99c895cb782..dcddd9f3d1318 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -44,7 +44,14 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.RecoverySource;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingHelper;
+import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.RemovalNotification;
@@ -55,12 +62,14 @@
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.XContentHelper;
@@ -69,9 +78,12 @@
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.cache.request.ShardRequestCache;
 import org.opensearch.index.query.TermQueryBuilder;
+import org.opensearch.index.seqno.RetentionLeaseSyncer;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardState;
+import org.opensearch.index.shard.IndexShardTestCase;
 import org.opensearch.index.shard.ShardNotFoundException;
+import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
 import org.opensearch.node.Node;
 import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
@@ -95,6 +107,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
 import static org.opensearch.indices.IndicesRequestCache.INDEX_DIMENSION_NAME;
 import static org.opensearch.indices.IndicesRequestCache.INDICES_CACHE_QUERY_SIZE;
 import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
@@ -1298,6 +1312,113 @@ public void testGetOrComputeConcurrentlyWithMultipleIndices() throws Exception {
         executorService.shutdownNow();
     }
 
+    public void testDeleteAndCreateIndexShardOnSameNodeAndVerifyStats() throws Exception {
+        threadPool = getThreadPool();
+        String indexName = "test1";
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        // Create a shard
+        IndexService indexService = createIndex(
+            indexName,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        Index idx = resolveIndex(indexName);
+        ShardRouting shardRouting = indicesService.indexService(idx).getShard(0).routingEntry();
+        IndexShard indexShard = indexService.getShard(0);
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        writer.addDocument(newDoc(1, "hack"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+        Loader loader = new Loader(reader, 0);
+
+        // Set clean interval to a high value as we will do it manually here.
+        IndicesRequestCache cache = getIndicesRequestCache(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY, TimeValue.timeValueMillis(100000))
+                .build()
+        );
+        IndicesService.IndexShardCacheEntity cacheEntity = new IndicesService.IndexShardCacheEntity(indexShard);
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "bar");
+        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+
+        // Cache some values for indexShard
+        BytesReference value = cache.getOrCompute(cacheEntity, loader, reader, getTermBytes());
+
+        // Verify response and stats.
+        assertEquals("foo", value.streamInput().readString());
+        RequestCacheStats stats = indexShard.requestCache().stats();
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, cache.count());
+        assertEquals(1, stats.getMissCount());
+        assertTrue(stats.getMemorySizeInBytes() > 0);
+
+        // Remove the shard making its cache entries stale
+        IOUtils.close(reader, writer, dir);
+        indexService.removeShard(0, "force");
+
+        // We again try to create a shard with same ShardId
+        ShardRouting newRouting = shardRouting;
+        String nodeId = newRouting.currentNodeId();
+        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom");
+        newRouting = newRouting.moveToUnassigned(unassignedInfo)
+            .updateUnassigned(unassignedInfo, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
+        newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
+        final DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
+        indexShard = indexService.createShard(
+            newRouting,
+            s -> {},
+            RetentionLeaseSyncer.EMPTY,
+            SegmentReplicationCheckpointPublisher.EMPTY,
+            null,
+            null,
+            localNode,
+            null,
+            DiscoveryNodes.builder().add(localNode).build()
+        );
+
+        // Verify that the new shard requestStats entries are empty.
+        stats = indexShard.requestCache().stats();
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, cache.count()); // Still contains the old indexShard stale entry
+        assertEquals(0, stats.getMissCount());
+        assertTrue(stats.getMemorySizeInBytes() == 0);
+        IndexShardTestCase.updateRoutingEntry(indexShard, newRouting);
+
+        // Now we cache again with new IndexShard(same shardId as older one).
+        dir = newDirectory();
+        writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        writer.addDocument(newDoc(1, "hack"));
+        reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+        loader = new Loader(reader, 0);
+        cacheEntity = new IndicesService.IndexShardCacheEntity(indexShard);
+        termQuery = new TermQueryBuilder("id", "bar");
+        termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        value = cache.getOrCompute(cacheEntity, loader, reader, getTermBytes());
+
+        // Assert response and stats. We verify that cache now has 2 entries, one for older/removed shard and other
+        // for the current shard.
+        assertEquals("foo", value.streamInput().readString());
+        stats = indexShard.requestCache().stats();
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(2, cache.count()); // One entry for older shard and other for the current shard.
+        assertEquals(1, stats.getMissCount());
+        assertTrue(stats.getMemorySizeInBytes() > 0);
+
+        // Trigger clean up of cache.
+        cache.cacheCleanupManager.cleanCache();
+        // Verify that cache still has entries for current shard and only removed older shards entries.
+        assertEquals(1, cache.count());
+
+        // Now make current indexShard entries stale as well.
+        reader.close();
+        // Trigger clean up of cache and verify that cache has no entries now.
+        cache.cacheCleanupManager.cleanCache();
+        assertEquals(0, cache.count());
+
+        IOUtils.close(reader, writer, dir, cache);
+    }
+
     public static String generateString(int length) {
         String characters = "abcdefghijklmnopqrstuvwxyz";
         StringBuilder sb = new StringBuilder(length);

From f28e673b85ab529e9c1f90e3ff95f668c53aaa6f Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Sat, 11 May 2024 02:54:30 +0800
Subject: [PATCH 26/52] Fix get field mapping API returns 404 error in mixed
 cluster with multiple versions (#13624)

* Fix get field mapping API returns 404 error in mixed cluster with multiple versions

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* modify changelog

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Add comment and remove unit test

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 CHANGELOG.md                                                | 1 +
 .../admin/indices/mapping/get/GetFieldMappingsResponse.java | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index db44887a0e59f..b24cc5347e768 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
+- Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index 86533f14e83e1..7edba143a72f0 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -52,6 +52,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -116,6 +117,11 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte
             String index = in.readString();
             if (in.getVersion().before(Version.V_2_0_0)) {
                 int typesSize = in.readVInt();
+                // if the requested field doesn't exist, type size in the received response from 1.x node is 0
+                if (typesSize == 0) {
+                    indexMapBuilder.put(index, Collections.emptyMap());
+                    continue;
+                }
                 if (typesSize != 1) {
                     throw new IllegalStateException("Expected single type but received [" + typesSize + "]");
                 }

From cd55bca3ee32d5754018e994b1b87799c1bffc32 Mon Sep 17 00:00:00 2001
From: Kunal Khatua <kkhatua@users.noreply.github.com>
Date: Fri, 10 May 2024 13:10:35 -0700
Subject: [PATCH 27/52] Add TODO for Updating API Spec (#13621)

Signed-off-by: Kunal Khatua <kkhatua@amazon.com>
Co-authored-by: Peter Nied <peternied@hotmail.com>
Co-authored-by: Peter Nied <petern@amazon.com>
---
 .github/pull_request_template.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 908a032bf833e..6281fa0af3e36 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -17,6 +17,7 @@ Resolves #[Issue number to be closed when this PR is merged]
   - [ ] All tests pass
 - [ ] New functionality has been documented.
   - [ ] New functionality has javadoc added
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
 - [ ] Failing checks are inspected and point to the corresponding known issue(s) (See: [Troubleshooting Failing Builds](../blob/main/CONTRIBUTING.md#troubleshooting-failing-builds))
 - [ ] Commits are signed per the DCO using --signoff
 - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))

From 83997fdc864eeb6fed4495d3365b086e376cbb74 Mon Sep 17 00:00:00 2001
From: Varun Lodaya <varunml@amazon.com>
Date: Mon, 13 May 2024 17:05:25 +0530
Subject: [PATCH 28/52] Updating security reach out email address (#13633)

Signed-off-by: varun-lodaya <varunml@amazon.com>
---
 README.md   | 2 +-
 SECURITY.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 748f8a366ecc8..aff18018960d5 100644
--- a/README.md
+++ b/README.md
@@ -45,7 +45,7 @@
 This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUCT.md). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq), or contact [opensource-codeofconduct@amazon.com](mailto:opensource-codeofconduct@amazon.com) with any additional questions or comments.
 
 ## Security
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue.
 
 ## License
 
diff --git a/SECURITY.md b/SECURITY.md
index b86292104335f..be4ac7463864a 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue.

From c328c18f69c08b21aa8a76af270b04a70c5a8069 Mon Sep 17 00:00:00 2001
From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com>
Date: Mon, 13 May 2024 17:06:18 +0530
Subject: [PATCH 29/52] [Remote Store] Permit backed futures to prevent
 timeouts during upload bursts (#12159)

Signed-off-by: vikasvb90 <vikasvb@amazon.com>
---
 .../s3/S3BlobStoreRepositoryTests.java        |  17 +-
 .../s3/GenericStatsMetricPublisher.java       |  90 ++++++
 .../repositories/s3/S3BlobContainer.java      |  62 +++-
 .../repositories/s3/S3BlobStore.java          |  34 ++-
 .../repositories/s3/S3Repository.java         |  68 ++++-
 .../repositories/s3/S3RepositoryPlugin.java   | 116 +++++++-
 .../s3/S3TransferRejectedException.java       |  20 ++
 .../s3/async/AsyncPartsHandler.java           | 135 ++++++---
 .../s3/async/AsyncTransferManager.java        | 134 ++++++---
 .../s3/async/SizeBasedBlockingQ.java          | 230 +++++++++++++++
 .../s3/async/TransferSemaphoresHolder.java    | 186 ++++++++++++
 .../repositories/s3/async/UploadRequest.java  |   2 +-
 .../s3/RepositoryCredentialsTests.java        |  17 +-
 .../s3/S3BlobContainerMockClientTests.java    |  88 +++++-
 .../s3/S3BlobContainerRetriesTests.java       |  51 +++-
 .../repositories/s3/S3RepositoryTests.java    |   6 +-
 .../s3/async/AsyncTransferManagerTests.java   |  12 +-
 .../s3/async/SizeBasedBlockingQTests.java     | 102 +++++++
 .../async/TransferSemaphoresHolderTests.java  | 276 ++++++++++++++++++
 .../common/blobstore/BlobStore.java           |   1 +
 .../blobstore/stream/write/WritePriority.java |   7 +-
 .../shard/RemoteStoreRefreshListener.java     |   6 +-
 .../index/store/RemoteDirectory.java          |  12 +-
 .../store/RemoteSegmentStoreDirectory.java    |   4 +-
 .../index/store/RemoteDirectoryTests.java     |   6 +-
 .../RemoteSegmentStoreDirectoryTests.java     |   4 +-
 26 files changed, 1555 insertions(+), 131 deletions(-)
 create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java
 create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java
 create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java
 create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java
 create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java
 create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java

diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index da2c6e8c1b0ee..21184380d54a9 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -249,7 +249,22 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, null, false) {
+            return new S3Repository(
+                metadata,
+                registry,
+                service,
+                clusterService,
+                recoverySettings,
+                null,
+                null,
+                null,
+                null,
+                null,
+                false,
+                null,
+                null,
+                null
+            ) {
 
                 @Override
                 public BlobStore blobStore() {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java
new file mode 100644
index 0000000000000..136fd68223354
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Generic stats of repository-s3 plugin.
+ */
+public class GenericStatsMetricPublisher {
+
+    private final AtomicLong normalPriorityQSize = new AtomicLong();
+    private final AtomicInteger normalPriorityPermits = new AtomicInteger();
+    private final AtomicLong lowPriorityQSize = new AtomicLong();
+    private final AtomicInteger lowPriorityPermits = new AtomicInteger();
+    private final long normalPriorityQCapacity;
+    private final int maxNormalPriorityPermits;
+    private final long lowPriorityQCapacity;
+    private final int maxLowPriorityPermits;
+
+    public GenericStatsMetricPublisher(
+        long normalPriorityQCapacity,
+        int maxNormalPriorityPermits,
+        long lowPriorityQCapacity,
+        int maxLowPriorityPermits
+    ) {
+        this.normalPriorityQCapacity = normalPriorityQCapacity;
+        this.maxNormalPriorityPermits = maxNormalPriorityPermits;
+        this.lowPriorityQCapacity = lowPriorityQCapacity;
+        this.maxLowPriorityPermits = maxLowPriorityPermits;
+    }
+
+    public void updateNormalPriorityQSize(long qSize) {
+        normalPriorityQSize.addAndGet(qSize);
+    }
+
+    public void updateLowPriorityQSize(long qSize) {
+        lowPriorityQSize.addAndGet(qSize);
+    }
+
+    public void updateNormalPermits(boolean increment) {
+        if (increment) {
+            normalPriorityPermits.incrementAndGet();
+        } else {
+            normalPriorityPermits.decrementAndGet();
+        }
+    }
+
+    public void updateLowPermits(boolean increment) {
+        if (increment) {
+            lowPriorityPermits.incrementAndGet();
+        } else {
+            lowPriorityPermits.decrementAndGet();
+        }
+    }
+
+    public long getNormalPriorityQSize() {
+        return normalPriorityQSize.get();
+    }
+
+    public int getAcquiredNormalPriorityPermits() {
+        return normalPriorityPermits.get();
+    }
+
+    public long getLowPriorityQSize() {
+        return lowPriorityQSize.get();
+    }
+
+    public int getAcquiredLowPriorityPermits() {
+        return lowPriorityPermits.get();
+    }
+
+    Map<String, Long> stats() {
+        final Map<String, Long> results = new HashMap<>();
+        results.put("NormalPriorityQUtilization", (normalPriorityQSize.get() * 100) / normalPriorityQCapacity);
+        results.put("LowPriorityQUtilization", (lowPriorityQSize.get() * 100) / lowPriorityQCapacity);
+        results.put("NormalPriorityPermitsUtilization", (normalPriorityPermits.get() * 100L) / maxNormalPriorityPermits);
+        results.put("LowPriorityPermitsUtilization", (lowPriorityPermits.get() * 100L) / maxLowPriorityPermits);
+        return results;
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index 14829a066ca3a..acf0c5e83a17b 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -90,6 +90,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 import org.opensearch.repositories.s3.async.UploadRequest;
 import org.opensearch.repositories.s3.utils.HttpRangeUtils;
 
@@ -218,7 +219,14 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
             writeContext.getMetadata()
         );
         try {
-            if (uploadRequest.getContentLength() > ByteSizeUnit.GB.toBytes(10) && blobStore.isRedirectLargeUploads()) {
+            // If file size is greater than the queue capacity than SizeBasedBlockingQ will always reject the upload.
+            // Therefore, redirecting it to slow client.
+            if ((uploadRequest.getWritePriority() == WritePriority.LOW
+                && blobStore.getLowPrioritySizeBasedBlockingQ().isMaxCapacityBelowContentLength(uploadRequest.getContentLength()) == false)
+                || (uploadRequest.getWritePriority() != WritePriority.HIGH
+                    && uploadRequest.getWritePriority() != WritePriority.URGENT
+                    && blobStore.getNormalPrioritySizeBasedBlockingQ()
+                        .isMaxCapacityBelowContentLength(uploadRequest.getContentLength()) == false)) {
                 StreamContext streamContext = SocketAccess.doPrivileged(
                     () -> writeContext.getStreamProvider(uploadRequest.getContentLength())
                 );
@@ -258,16 +266,30 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
                 } else {
                     s3AsyncClient = amazonS3Reference.get().client();
                 }
-                CompletableFuture<Void> completableFuture = blobStore.getAsyncTransferManager()
-                    .uploadObject(s3AsyncClient, uploadRequest, streamContext, blobStore.getStatsMetricPublisher());
-                completableFuture.whenComplete((response, throwable) -> {
-                    if (throwable == null) {
-                        completionListener.onResponse(response);
-                    } else {
-                        Exception ex = throwable instanceof Error ? new Exception(throwable) : (Exception) throwable;
-                        completionListener.onFailure(ex);
-                    }
-                });
+
+                if (writeContext.getWritePriority() == WritePriority.URGENT
+                    || writeContext.getWritePriority() == WritePriority.HIGH
+                    || blobStore.isPermitBackedTransferEnabled() == false) {
+                    createFileCompletableFuture(s3AsyncClient, uploadRequest, streamContext, completionListener);
+                } else if (writeContext.getWritePriority() == WritePriority.LOW) {
+                    blobStore.getLowPrioritySizeBasedBlockingQ()
+                        .produce(
+                            new SizeBasedBlockingQ.Item(
+                                writeContext.getFileSize(),
+                                () -> createFileCompletableFuture(s3AsyncClient, uploadRequest, streamContext, completionListener)
+                            )
+                        );
+                } else if (writeContext.getWritePriority() == WritePriority.NORMAL) {
+                    blobStore.getNormalPrioritySizeBasedBlockingQ()
+                        .produce(
+                            new SizeBasedBlockingQ.Item(
+                                writeContext.getFileSize(),
+                                () -> createFileCompletableFuture(s3AsyncClient, uploadRequest, streamContext, completionListener)
+                            )
+                        );
+                } else {
+                    throw new IllegalStateException("Cannot perform upload for other priority types.");
+                }
             }
         } catch (Exception e) {
             logger.info("exception error from blob container for file {}", writeContext.getFileName());
@@ -275,6 +297,24 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
         }
     }
 
+    private CompletableFuture<Void> createFileCompletableFuture(
+        S3AsyncClient s3AsyncClient,
+        UploadRequest uploadRequest,
+        StreamContext streamContext,
+        ActionListener<Void> completionListener
+    ) {
+        CompletableFuture<Void> completableFuture = blobStore.getAsyncTransferManager()
+            .uploadObject(s3AsyncClient, uploadRequest, streamContext, blobStore.getStatsMetricPublisher());
+        return completableFuture.whenComplete((response, throwable) -> {
+            if (throwable == null) {
+                completionListener.onResponse(response);
+            } else {
+                Exception ex = throwable instanceof Error ? new Exception(throwable) : (Exception) throwable;
+                completionListener.onFailure(ex);
+            }
+        });
+    }
+
     @ExperimentalApi
     @Override
     public void readBlobAsync(String blobName, ActionListener<ReadContext> listener) {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index fc70fbb0db00e..de815f9202f44 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -45,6 +45,7 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -56,6 +57,7 @@
 import static org.opensearch.repositories.s3.S3Repository.BUFFER_SIZE_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
 import static org.opensearch.repositories.s3.S3Repository.CANNED_ACL_SETTING;
+import static org.opensearch.repositories.s3.S3Repository.PERMIT_BACKED_TRANSFER_ENABLED;
 import static org.opensearch.repositories.s3.S3Repository.REDIRECT_LARGE_S3_UPLOAD;
 import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING;
@@ -77,6 +79,8 @@ class S3BlobStore implements BlobStore {
 
     private volatile boolean uploadRetryEnabled;
 
+    private volatile boolean permitBackedTransferEnabled;
+
     private volatile boolean serverSideEncryption;
 
     private volatile ObjectCannedACL cannedACL;
@@ -94,6 +98,9 @@ class S3BlobStore implements BlobStore {
     private final AsyncExecutorContainer priorityExecutorBuilder;
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final boolean multipartUploadEnabled;
+    private final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
+    private final GenericStatsMetricPublisher genericStatsMetricPublisher;
 
     S3BlobStore(
         S3Service service,
@@ -109,7 +116,10 @@ class S3BlobStore implements BlobStore {
         AsyncTransferManager asyncTransferManager,
         AsyncExecutorContainer urgentExecutorBuilder,
         AsyncExecutorContainer priorityExecutorBuilder,
-        AsyncExecutorContainer normalExecutorBuilder
+        AsyncExecutorContainer normalExecutorBuilder,
+        SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
+        SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
+        GenericStatsMetricPublisher genericStatsMetricPublisher
     ) {
         this.service = service;
         this.s3AsyncService = s3AsyncService;
@@ -128,6 +138,10 @@ class S3BlobStore implements BlobStore {
         // Settings to initialize blobstore with.
         this.redirectLargeUploads = REDIRECT_LARGE_S3_UPLOAD.get(repositoryMetadata.settings());
         this.uploadRetryEnabled = UPLOAD_RETRY_ENABLED.get(repositoryMetadata.settings());
+        this.normalPrioritySizeBasedBlockingQ = normalPrioritySizeBasedBlockingQ;
+        this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
+        this.genericStatsMetricPublisher = genericStatsMetricPublisher;
+        this.permitBackedTransferEnabled = PERMIT_BACKED_TRANSFER_ENABLED.get(repositoryMetadata.settings());
     }
 
     @Override
@@ -141,6 +155,7 @@ public void reload(RepositoryMetadata repositoryMetadata) {
         this.bulkDeletesSize = BULK_DELETE_SIZE.get(repositoryMetadata.settings());
         this.redirectLargeUploads = REDIRECT_LARGE_S3_UPLOAD.get(repositoryMetadata.settings());
         this.uploadRetryEnabled = UPLOAD_RETRY_ENABLED.get(repositoryMetadata.settings());
+        this.permitBackedTransferEnabled = PERMIT_BACKED_TRANSFER_ENABLED.get(repositoryMetadata.settings());
     }
 
     @Override
@@ -168,6 +183,10 @@ public boolean isUploadRetryEnabled() {
         return uploadRetryEnabled;
     }
 
+    public boolean isPermitBackedTransferEnabled() {
+        return permitBackedTransferEnabled;
+    }
+
     public String bucket() {
         return bucket;
     }
@@ -184,6 +203,14 @@ public int getBulkDeletesSize() {
         return bulkDeletesSize;
     }
 
+    public SizeBasedBlockingQ getNormalPrioritySizeBasedBlockingQ() {
+        return normalPrioritySizeBasedBlockingQ;
+    }
+
+    public SizeBasedBlockingQ getLowPrioritySizeBasedBlockingQ() {
+        return lowPrioritySizeBasedBlockingQ;
+    }
+
     @Override
     public BlobContainer blobContainer(BlobPath path) {
         return new S3BlobContainer(path, this);
@@ -201,7 +228,9 @@ public void close() throws IOException {
 
     @Override
     public Map<String, Long> stats() {
-        return statsMetricPublisher.getStats().toMap();
+        Map<String, Long> stats = statsMetricPublisher.getStats().toMap();
+        stats.putAll(genericStatsMetricPublisher.stats());
+        return stats;
     }
 
     @Override
@@ -211,6 +240,7 @@ public Map<Metric, Map<String, Long>> extendedStats() {
         }
         Map<Metric, Map<String, Long>> extendedStats = new HashMap<>();
         statsMetricPublisher.getExtendedStats().forEach((k, v) -> extendedStats.put(k, v.toMap()));
+        extendedStats.put(Metric.GENERIC_STATS, genericStatsMetricPublisher.stats());
         return extendedStats;
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
index f7772a57c9afd..01b75c0b915f2 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
@@ -49,6 +49,7 @@
 import org.opensearch.common.settings.SecureSetting;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.settings.SecureString;
@@ -63,6 +64,7 @@
 import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository;
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 import org.opensearch.snapshots.SnapshotId;
 import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.threadpool.Scheduler;
@@ -156,6 +158,15 @@ class S3Repository extends MeteredBlobStoreRepository {
         Setting.Property.NodeScope
     );
 
+    /**
+     * Whether large uploads need to be redirected to slow sync s3 client.
+     */
+    static final Setting<Boolean> PERMIT_BACKED_TRANSFER_ENABLED = Setting.boolSetting(
+        "permit_backed_transfer_enabled",
+        true,
+        Setting.Property.NodeScope
+    );
+
     /**
      * Whether retry on uploads are enabled. This setting wraps inputstream with buffered stream to enable retries.
      */
@@ -193,6 +204,37 @@ class S3Repository extends MeteredBlobStoreRepository {
         true,
         Setting.Property.NodeScope
     );
+    /**
+     * Percentage of total available permits to be available for priority transfers.
+     */
+    public static Setting<Integer> S3_PRIORITY_PERMIT_ALLOCATION_PERCENT = Setting.intSetting(
+        "s3_priority_permit_alloc_perc",
+        70,
+        21,
+        80,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Duration in minutes to wait for a permit in case no permit is available.
+     */
+    public static Setting<Integer> S3_PERMIT_WAIT_DURATION_MIN = Setting.intSetting(
+        "s3_permit_wait_duration_min",
+        5,
+        1,
+        10,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Number of transfer queue consumers
+     */
+    public static Setting<Integer> S3_TRANSFER_QUEUE_CONSUMERS = new Setting<>(
+        "s3_transfer_queue_consumers",
+        (s) -> Integer.toString(Math.max(5, OpenSearchExecutors.allocatedProcessors(s) * 2)),
+        (s) -> Setting.parseInt(s, 5, "s3_transfer_queue_consumers"),
+        Setting.Property.NodeScope
+    );
 
     /**
      * Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
@@ -252,6 +294,9 @@ class S3Repository extends MeteredBlobStoreRepository {
     private final AsyncExecutorContainer priorityExecutorBuilder;
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final Path pluginConfigPath;
+    private final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
+    private final GenericStatsMetricPublisher genericStatsMetricPublisher;
 
     private volatile int bulkDeletesSize;
 
@@ -267,7 +312,10 @@ class S3Repository extends MeteredBlobStoreRepository {
         final AsyncExecutorContainer priorityExecutorBuilder,
         final AsyncExecutorContainer normalExecutorBuilder,
         final S3AsyncService s3AsyncService,
-        final boolean multipartUploadEnabled
+        final boolean multipartUploadEnabled,
+        final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
+        final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
+        final GenericStatsMetricPublisher genericStatsMetricPublisher
     ) {
         this(
             metadata,
@@ -281,7 +329,10 @@ class S3Repository extends MeteredBlobStoreRepository {
             normalExecutorBuilder,
             s3AsyncService,
             multipartUploadEnabled,
-            Path.of("")
+            Path.of(""),
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
@@ -300,7 +351,10 @@ class S3Repository extends MeteredBlobStoreRepository {
         final AsyncExecutorContainer normalExecutorBuilder,
         final S3AsyncService s3AsyncService,
         final boolean multipartUploadEnabled,
-        Path pluginConfigPath
+        Path pluginConfigPath,
+        final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
+        final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
+        final GenericStatsMetricPublisher genericStatsMetricPublisher
     ) {
         super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildLocation(metadata));
         this.service = service;
@@ -311,6 +365,9 @@ class S3Repository extends MeteredBlobStoreRepository {
         this.urgentExecutorBuilder = urgentExecutorBuilder;
         this.priorityExecutorBuilder = priorityExecutorBuilder;
         this.normalExecutorBuilder = normalExecutorBuilder;
+        this.normalPrioritySizeBasedBlockingQ = normalPrioritySizeBasedBlockingQ;
+        this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
+        this.genericStatsMetricPublisher = genericStatsMetricPublisher;
 
         validateRepositoryMetadata(metadata);
         readRepositoryMetadata();
@@ -373,7 +430,10 @@ protected S3BlobStore createBlobStore() {
             asyncUploadUtils,
             urgentExecutorBuilder,
             priorityExecutorBuilder,
-            normalExecutorBuilder
+            normalExecutorBuilder,
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
index e7d2a4d024e60..110d91bfbd822 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
@@ -41,6 +41,9 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
@@ -53,6 +56,8 @@
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
+import org.opensearch.repositories.s3.async.TransferSemaphoresHolder;
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.ExecutorBuilder;
 import org.opensearch.threadpool.FixedExecutorBuilder;
@@ -69,6 +74,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 /**
@@ -82,6 +89,8 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
     private static final String PRIORITY_STREAM_READER = "priority_stream_reader";
     private static final String FUTURE_COMPLETION = "future_completion";
     private static final String STREAM_READER = "stream_reader";
+    private static final String LOW_TRANSFER_QUEUE_CONSUMER = "low_transfer_queue_consumer";
+    private static final String NORMAL_TRANSFER_QUEUE_CONSUMER = "normal_transfer_queue_consumer";
 
     protected final S3Service service;
     private final S3AsyncService s3AsyncService;
@@ -91,6 +100,12 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
     private AsyncExecutorContainer urgentExecutorBuilder;
     private AsyncExecutorContainer priorityExecutorBuilder;
     private AsyncExecutorContainer normalExecutorBuilder;
+    private ExecutorService lowTransferQConsumerService;
+    private ExecutorService normalTransferQConsumerService;
+    private SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
+    private TransferSemaphoresHolder transferSemaphoresHolder;
+    private GenericStatsMetricPublisher genericStatsMetricPublisher;
 
     public S3RepositoryPlugin(final Settings settings, final Path configPath) {
         this(settings, configPath, new S3Service(configPath), new S3AsyncService(configPath));
@@ -120,9 +135,36 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
                 TimeValue.timeValueMinutes(5)
             )
         );
+        executorBuilders.add(
+            new FixedExecutorBuilder(
+                settings,
+                LOW_TRANSFER_QUEUE_CONSUMER,
+                lowPriorityTransferQConsumers(settings),
+                10,
+                "thread_pool." + LOW_TRANSFER_QUEUE_CONSUMER
+            )
+        );
+        executorBuilders.add(
+            new FixedExecutorBuilder(
+                settings,
+                NORMAL_TRANSFER_QUEUE_CONSUMER,
+                normalPriorityTransferQConsumers(settings),
+                10,
+                "thread_pool." + NORMAL_TRANSFER_QUEUE_CONSUMER
+            )
+        );
         return executorBuilders;
     }
 
+    private int lowPriorityTransferQConsumers(Settings settings) {
+        double lowPriorityAllocation = ((double) (100 - S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT.get(settings))) / 100;
+        return Math.max(2, (int) (lowPriorityAllocation * S3Repository.S3_TRANSFER_QUEUE_CONSUMERS.get(settings)));
+    }
+
+    private int normalPriorityTransferQConsumers(Settings settings) {
+        return S3Repository.S3_TRANSFER_QUEUE_CONSUMERS.get(settings);
+    }
+
     static int halfNumberOfProcessors(int numberOfProcessors) {
         return (numberOfProcessors + 1) / 2;
     }
@@ -189,7 +231,67 @@ public Collection<Object> createComponents(
             threadPool.executor(STREAM_READER),
             new AsyncTransferEventLoopGroup(normalEventLoopThreads)
         );
-        return Collections.emptyList();
+
+        this.lowTransferQConsumerService = threadPool.executor(LOW_TRANSFER_QUEUE_CONSUMER);
+        this.normalTransferQConsumerService = threadPool.executor(NORMAL_TRANSFER_QUEUE_CONSUMER);
+
+        // High number of permit allocation because each op acquiring permit performs disk IO, computation and network IO.
+        int availablePermits = Math.max(allocatedProcessors(clusterService.getSettings()) * 4, 10);
+        double priorityPermitAllocation = ((double) S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT.get(clusterService.getSettings()))
+            / 100;
+        int normalPriorityPermits = (int) (priorityPermitAllocation * availablePermits);
+        int lowPriorityPermits = availablePermits - normalPriorityPermits;
+
+        int normalPriorityConsumers = normalPriorityTransferQConsumers(clusterService.getSettings());
+        int lowPriorityConsumers = lowPriorityTransferQConsumers(clusterService.getSettings());
+
+        ByteSizeValue normalPriorityQCapacity = new ByteSizeValue(normalPriorityConsumers * 10L, ByteSizeUnit.GB);
+        ByteSizeValue lowPriorityQCapacity = new ByteSizeValue(lowPriorityConsumers * 20L, ByteSizeUnit.GB);
+
+        this.genericStatsMetricPublisher = new GenericStatsMetricPublisher(
+            normalPriorityQCapacity.getBytes(),
+            normalPriorityPermits,
+            lowPriorityQCapacity.getBytes(),
+            lowPriorityPermits
+        );
+
+        this.normalPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            normalPriorityQCapacity,
+            normalTransferQConsumerService,
+            normalPriorityConsumers,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+
+        LowPrioritySizeBasedBlockingQ lowPrioritySizeBasedBlockingQ = new LowPrioritySizeBasedBlockingQ(
+            lowPriorityQCapacity,
+            lowTransferQConsumerService,
+            lowPriorityConsumers,
+            genericStatsMetricPublisher
+        );
+        this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
+        this.transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPriorityPermits,
+            lowPriorityPermits,
+            S3Repository.S3_PERMIT_WAIT_DURATION_MIN.get(clusterService.getSettings()),
+            TimeUnit.MINUTES,
+            genericStatsMetricPublisher
+        );
+
+        return CollectionUtils.arrayAsArrayList(this.normalPrioritySizeBasedBlockingQ, lowPrioritySizeBasedBlockingQ);
+    }
+
+    // New class because in core, components are injected via guice only by instance creation due to which
+    // same binding types fail.
+    private static final class LowPrioritySizeBasedBlockingQ extends SizeBasedBlockingQ {
+        public LowPrioritySizeBasedBlockingQ(
+            ByteSizeValue capacity,
+            ExecutorService executorService,
+            int consumers,
+            GenericStatsMetricPublisher genericStatsMetricPublisher
+        ) {
+            super(capacity, executorService, consumers, genericStatsMetricPublisher, QueueEventType.LOW);
+        }
     }
 
     // proxy method for testing
@@ -204,7 +306,8 @@ protected S3Repository createRepository(
             S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.get(clusterService.getSettings()).getBytes(),
             normalExecutorBuilder.getStreamReader(),
             priorityExecutorBuilder.getStreamReader(),
-            urgentExecutorBuilder.getStreamReader()
+            urgentExecutorBuilder.getStreamReader(),
+            transferSemaphoresHolder
         );
         return new S3Repository(
             metadata,
@@ -218,7 +321,10 @@ protected S3Repository createRepository(
             normalExecutorBuilder,
             s3AsyncService,
             S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING.get(clusterService.getSettings()),
-            configPath
+            configPath,
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
@@ -263,7 +369,9 @@ public List<Setting<?>> getSettings() {
             S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING,
             S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING,
             S3Repository.REDIRECT_LARGE_S3_UPLOAD,
-            S3Repository.UPLOAD_RETRY_ENABLED
+            S3Repository.UPLOAD_RETRY_ENABLED,
+            S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT,
+            S3Repository.PERMIT_BACKED_TRANSFER_ENABLED
         );
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java
new file mode 100644
index 0000000000000..c9fa93ea0f5c3
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import org.opensearch.OpenSearchException;
+
+/**
+ * Thrown when transfer event is rejected due to breach in event queue size.
+ */
+public class S3TransferRejectedException extends OpenSearchException {
+    public S3TransferRejectedException(String msg) {
+        super(msg);
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
index b4c4ed0ecaa75..4c95a0ffc5ec3 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
@@ -23,6 +23,7 @@
 import org.opensearch.common.StreamContext;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.io.InputStreamContainer;
+import org.opensearch.repositories.s3.S3TransferRejectedException;
 import org.opensearch.repositories.s3.SocketAccess;
 import org.opensearch.repositories.s3.StatsMetricPublisher;
 import org.opensearch.repositories.s3.io.CheckedContainer;
@@ -34,6 +35,8 @@
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
@@ -41,7 +44,7 @@
  */
 public class AsyncPartsHandler {
 
-    private static Logger log = LogManager.getLogger(AsyncPartsHandler.class);
+    private static final Logger log = LogManager.getLogger(AsyncPartsHandler.class);
 
     /**
      * Uploads parts of the upload multipart request*
@@ -55,9 +58,10 @@ public class AsyncPartsHandler {
      * @param completedParts Reference of completed parts
      * @param inputStreamContainers Checksum containers
      * @param statsMetricPublisher sdk metric publisher
+     * @param maxRetryablePartSize Max content size which can be used for retries in buffered streams.
      * @return list of completable futures
-     * @throws IOException thrown in case of an IO error
      */
+    @SuppressWarnings({ "rawtypes", "unchecked" })
     public static List<CompletableFuture<CompletedPart>> uploadParts(
         S3AsyncClient s3AsyncClient,
         ExecutorService executorService,
@@ -69,35 +73,52 @@ public static List<CompletableFuture<CompletedPart>> uploadParts(
         AtomicReferenceArray<CompletedPart> completedParts,
         AtomicReferenceArray<CheckedContainer> inputStreamContainers,
         StatsMetricPublisher statsMetricPublisher,
-        boolean uploadRetryEnabled
-    ) throws IOException {
+        boolean uploadRetryEnabled,
+        TransferSemaphoresHolder transferSemaphoresHolder,
+        long maxRetryablePartSize
+    ) throws InterruptedException {
         List<CompletableFuture<CompletedPart>> futures = new ArrayList<>();
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
         for (int partIdx = 0; partIdx < streamContext.getNumberOfParts(); partIdx++) {
-            InputStreamContainer inputStreamContainer = streamContext.provideStream(partIdx);
-            inputStreamContainers.set(partIdx, new CheckedContainer(inputStreamContainer.getContentLength()));
-            UploadPartRequest.Builder uploadPartRequestBuilder = UploadPartRequest.builder()
-                .bucket(uploadRequest.getBucket())
-                .partNumber(partIdx + 1)
-                .key(uploadRequest.getKey())
-                .uploadId(uploadId)
-                .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.multipartUploadMetricCollector))
-                .contentLength(inputStreamContainer.getContentLength());
-            if (uploadRequest.doRemoteDataIntegrityCheck()) {
-                uploadPartRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
-            }
-            uploadPart(
-                s3AsyncClient,
-                executorService,
-                priorityExecutorService,
-                urgentExecutorService,
-                completedParts,
-                inputStreamContainers,
-                futures,
-                uploadPartRequestBuilder.build(),
-                inputStreamContainer,
-                uploadRequest,
-                uploadRetryEnabled
+            Semaphore semaphore = maybeAcquireSemaphore(
+                transferSemaphoresHolder,
+                requestContext,
+                uploadRequest.getWritePriority(),
+                uploadRequest.getKey()
             );
+            try {
+                InputStreamContainer inputStreamContainer = streamContext.provideStream(partIdx);
+                inputStreamContainers.set(partIdx, new CheckedContainer(inputStreamContainer.getContentLength()));
+                UploadPartRequest.Builder uploadPartRequestBuilder = UploadPartRequest.builder()
+                    .bucket(uploadRequest.getBucket())
+                    .partNumber(partIdx + 1)
+                    .key(uploadRequest.getKey())
+                    .uploadId(uploadId)
+                    .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.multipartUploadMetricCollector))
+                    .contentLength(inputStreamContainer.getContentLength());
+                if (uploadRequest.doRemoteDataIntegrityCheck()) {
+                    uploadPartRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
+                }
+                uploadPart(
+                    s3AsyncClient,
+                    executorService,
+                    priorityExecutorService,
+                    urgentExecutorService,
+                    completedParts,
+                    inputStreamContainers,
+                    futures,
+                    uploadPartRequestBuilder.build(),
+                    inputStreamContainer,
+                    uploadRequest,
+                    uploadRetryEnabled,
+                    maxRetryablePartSize,
+                    semaphore
+                );
+            } catch (Exception ex) {
+                if (semaphore != null) {
+                    semaphore.release();
+                }
+            }
         }
 
         return futures;
@@ -137,14 +158,54 @@ public static InputStream maybeRetryInputStream(
         InputStream inputStream,
         WritePriority writePriority,
         boolean uploadRetryEnabled,
-        long contentLength
+        long contentLength,
+        long maxRetryablePartSize
     ) {
-        if (uploadRetryEnabled == true && (writePriority == WritePriority.HIGH || writePriority == WritePriority.URGENT)) {
-            return new BufferedInputStream(inputStream, (int) (contentLength + 1));
+        // Since we are backing uploads with limited permits, it is ok to use buffered stream. Maximum in-memory buffer
+        // would be (max permits * maxRetryablePartSize) excluding urgent
+        if (uploadRetryEnabled == true
+            && (contentLength <= maxRetryablePartSize || writePriority == WritePriority.HIGH || writePriority == WritePriority.URGENT)) {
+            return new UploadTrackedBufferedInputStream(inputStream, (int) (contentLength + 1));
         }
         return inputStream;
     }
 
+    public static Semaphore maybeAcquireSemaphore(
+        TransferSemaphoresHolder transferSemaphoresHolder,
+        TransferSemaphoresHolder.RequestContext requestContext,
+        WritePriority writePriority,
+        String file
+    ) throws InterruptedException {
+        final TransferSemaphoresHolder.TypeSemaphore semaphore;
+        if (writePriority != WritePriority.HIGH && writePriority != WritePriority.URGENT) {
+            semaphore = transferSemaphoresHolder.acquirePermit(writePriority, requestContext);
+            if (semaphore == null) {
+                throw new S3TransferRejectedException("Permit not available for transfer of file " + file);
+            }
+        } else {
+            semaphore = null;
+        }
+
+        return semaphore;
+    }
+
+    /**
+     * Overridden stream to identify upload streams among all buffered stream instances for triaging.
+     */
+    static class UploadTrackedBufferedInputStream extends BufferedInputStream {
+        AtomicBoolean closed = new AtomicBoolean();
+
+        public UploadTrackedBufferedInputStream(InputStream in, int length) {
+            super(in, length);
+        }
+
+        @Override
+        public void close() throws IOException {
+            super.close();
+            closed.set(true);
+        }
+    }
+
     private static void uploadPart(
         S3AsyncClient s3AsyncClient,
         ExecutorService executorService,
@@ -156,8 +217,11 @@ private static void uploadPart(
         UploadPartRequest uploadPartRequest,
         InputStreamContainer inputStreamContainer,
         UploadRequest uploadRequest,
-        boolean uploadRetryEnabled
+        boolean uploadRetryEnabled,
+        long maxRetryablePartSize,
+        Semaphore semaphore
     ) {
+
         Integer partNumber = uploadPartRequest.partNumber();
 
         ExecutorService streamReadExecutor;
@@ -173,7 +237,8 @@ private static void uploadPart(
             inputStreamContainer.getInputStream(),
             uploadRequest.getWritePriority(),
             uploadRetryEnabled,
-            uploadPartRequest.contentLength()
+            uploadPartRequest.contentLength(),
+            maxRetryablePartSize
         );
         CompletableFuture<UploadPartResponse> uploadPartResponseFuture = SocketAccess.doPrivileged(
             () -> s3AsyncClient.uploadPart(
@@ -183,6 +248,10 @@ private static void uploadPart(
         );
 
         CompletableFuture<CompletedPart> convertFuture = uploadPartResponseFuture.whenComplete((resp, throwable) -> {
+            if (semaphore != null) {
+                semaphore.release();
+            }
+
             try {
                 inputStream.close();
             } catch (IOException ex) {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index 80538059d17b8..0f9bf3be77d73 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -21,6 +21,7 @@
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Exception;
 import software.amazon.awssdk.utils.CollectionUtils;
 import software.amazon.awssdk.utils.CompletableFutureUtils;
@@ -48,6 +49,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
@@ -64,6 +66,10 @@ public final class AsyncTransferManager {
     private final ExecutorService priorityExecutorService;
     private final ExecutorService urgentExecutorService;
     private final long minimumPartSize;
+    private final long maxRetryablePartSize;
+
+    @SuppressWarnings("rawtypes")
+    private final TransferSemaphoresHolder transferSemaphoresHolder;
 
     /**
      * The max number of parts on S3 side is 10,000
@@ -74,19 +80,22 @@ public final class AsyncTransferManager {
      * Construct a new object of AsyncTransferManager
      *
      * @param minimumPartSize         The minimum part size for parallel multipart uploads
-     * @param executorService         The stream reader {@link ExecutorService} for normal priority uploads
-     * @param priorityExecutorService The stream read {@link ExecutorService} for high priority uploads
      */
+    @SuppressWarnings("rawtypes")
     public AsyncTransferManager(
         long minimumPartSize,
         ExecutorService executorService,
         ExecutorService priorityExecutorService,
-        ExecutorService urgentExecutorService
+        ExecutorService urgentExecutorService,
+        TransferSemaphoresHolder transferSemaphoresHolder
     ) {
         this.executorService = executorService;
         this.priorityExecutorService = priorityExecutorService;
         this.minimumPartSize = minimumPartSize;
+        // 10% buffer to allow additional metadata size in content such as encryption.
+        this.maxRetryablePartSize = (long) (minimumPartSize + 0.1 * minimumPartSize);
         this.urgentExecutorService = urgentExecutorService;
+        this.transferSemaphoresHolder = transferSemaphoresHolder;
     }
 
     /**
@@ -108,7 +117,21 @@ public CompletableFuture<Void> uploadObject(
         try {
             if (streamContext.getNumberOfParts() == 1) {
                 log.debug(() -> "Starting the upload as a single upload part request");
-                uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext.provideStream(0), returnFuture, statsMetricPublisher);
+                TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+                Semaphore semaphore = AsyncPartsHandler.maybeAcquireSemaphore(
+                    transferSemaphoresHolder,
+                    requestContext,
+                    uploadRequest.getWritePriority(),
+                    uploadRequest.getKey()
+                );
+                try {
+                    uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext, returnFuture, statsMetricPublisher, semaphore);
+                } catch (Exception ex) {
+                    if (semaphore != null) {
+                        semaphore.release();
+                    }
+                    throw ex;
+                }
             } else {
                 log.debug(() -> "Starting the upload as multipart upload request");
                 uploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture, statsMetricPublisher);
@@ -146,21 +169,19 @@ private void uploadInParts(
         // Ensure cancellations are forwarded to the createMultipartUploadFuture future
         CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture);
 
-        createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> {
-            if (throwable != null) {
-                handleException(returnFuture, () -> "Failed to initiate multipart upload", throwable);
-            } else {
-                log.debug(() -> "Initiated new multipart upload, uploadId: " + createMultipartUploadResponse.uploadId());
-                doUploadInParts(
-                    s3AsyncClient,
-                    uploadRequest,
-                    streamContext,
-                    returnFuture,
-                    createMultipartUploadResponse.uploadId(),
-                    statsMetricPublisher
-                );
-            }
-        });
+        String uploadId;
+        try {
+            // Block main thread here so that upload of parts doesn't get executed in future completion thread.
+            // We should never execute latent operation like acquisition of permit in future completion pool.
+            CreateMultipartUploadResponse createMultipartUploadResponse = createMultipartUploadFuture.get();
+            uploadId = createMultipartUploadResponse.uploadId();
+            log.debug(() -> "Initiated new multipart upload, uploadId: " + createMultipartUploadResponse.uploadId());
+        } catch (Exception ex) {
+            handleException(returnFuture, () -> "Failed to initiate multipart upload", ex);
+            return;
+        }
+
+        doUploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture, uploadId, statsMetricPublisher);
     }
 
     private void doUploadInParts(
@@ -189,7 +210,9 @@ private void doUploadInParts(
                 completedParts,
                 inputStreamContainers,
                 statsMetricPublisher,
-                uploadRequest.isUploadRetryEnabled()
+                uploadRequest.isUploadRetryEnabled(),
+                transferSemaphoresHolder,
+                maxRetryablePartSize
             );
         } catch (Exception ex) {
             try {
@@ -320,12 +343,14 @@ public long calculateOptimalPartSize(long contentLengthOfSource, WritePriority w
         return (long) Math.max(optimalPartSize, minimumPartSize);
     }
 
+    @SuppressWarnings("unchecked")
     private void uploadInOneChunk(
         S3AsyncClient s3AsyncClient,
         UploadRequest uploadRequest,
-        InputStreamContainer inputStreamContainer,
+        StreamContext streamContext,
         CompletableFuture<Void> returnFuture,
-        StatsMetricPublisher statsMetricPublisher
+        StatsMetricPublisher statsMetricPublisher,
+        Semaphore semaphore
     ) {
         PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
             .bucket(uploadRequest.getBucket())
@@ -340,6 +365,7 @@ private void uploadInOneChunk(
             putObjectRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
             putObjectRequestBuilder.checksumCRC32(base64StringFromLong(uploadRequest.getExpectedChecksum()));
         }
+        PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
         ExecutorService streamReadExecutor;
         if (uploadRequest.getWritePriority() == WritePriority.URGENT) {
             streamReadExecutor = urgentExecutorService;
@@ -349,25 +375,33 @@ private void uploadInOneChunk(
             streamReadExecutor = executorService;
         }
 
-        InputStream inputStream = AsyncPartsHandler.maybeRetryInputStream(
-            inputStreamContainer.getInputStream(),
-            uploadRequest.getWritePriority(),
-            uploadRequest.isUploadRetryEnabled(),
-            uploadRequest.getContentLength()
-        );
-        CompletableFuture<Void> putObjectFuture = SocketAccess.doPrivileged(
-            () -> s3AsyncClient.putObject(
-                putObjectRequestBuilder.build(),
-                AsyncRequestBody.fromInputStream(inputStream, inputStreamContainer.getContentLength(), streamReadExecutor)
-            ).handle((resp, throwable) -> {
-                try {
-                    inputStream.close();
-                } catch (IOException e) {
-                    log.error(
-                        () -> new ParameterizedMessage("Failed to close stream while uploading single file {}.", uploadRequest.getKey()),
-                        e
-                    );
-                }
+        CompletableFuture<Void> putObjectFuture = SocketAccess.doPrivileged(() -> {
+            InputStream inputStream = null;
+            CompletableFuture<PutObjectResponse> putObjectRespFuture;
+            try {
+                InputStreamContainer inputStreamContainer = streamContext.provideStream(0);
+                inputStream = AsyncPartsHandler.maybeRetryInputStream(
+                    inputStreamContainer.getInputStream(),
+                    uploadRequest.getWritePriority(),
+                    uploadRequest.isUploadRetryEnabled(),
+                    uploadRequest.getContentLength(),
+                    maxRetryablePartSize
+                );
+                AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(
+                    inputStream,
+                    inputStreamContainer.getContentLength(),
+                    streamReadExecutor
+                );
+                putObjectRespFuture = s3AsyncClient.putObject(putObjectRequest, asyncRequestBody);
+            } catch (Exception e) {
+                releaseResourcesSafely(semaphore, inputStream, uploadRequest.getKey());
+                return CompletableFuture.failedFuture(e);
+            }
+
+            InputStream finalInputStream = inputStream;
+            return putObjectRespFuture.handle((resp, throwable) -> {
+                releaseResourcesSafely(semaphore, finalInputStream, uploadRequest.getKey());
+
                 if (throwable != null) {
                     Throwable unwrappedThrowable = ExceptionsHelper.unwrap(throwable, S3Exception.class);
                     if (unwrappedThrowable != null) {
@@ -395,13 +429,27 @@ private void uploadInOneChunk(
                 }
 
                 return null;
-            })
-        );
+            });
+        });
 
         CompletableFutureUtils.forwardExceptionTo(returnFuture, putObjectFuture);
         CompletableFutureUtils.forwardResultTo(putObjectFuture, returnFuture);
     }
 
+    private void releaseResourcesSafely(Semaphore semaphore, InputStream inputStream, String file) {
+        if (semaphore != null) {
+            semaphore.release();
+        }
+
+        if (inputStream != null) {
+            try {
+                inputStream.close();
+            } catch (IOException e) {
+                log.error(() -> new ParameterizedMessage("Failed to close stream while uploading single file {}.", file), e);
+            }
+        }
+    }
+
     private void deleteUploadedObject(S3AsyncClient s3AsyncClient, UploadRequest uploadRequest) {
         DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
             .bucket(uploadRequest.getBucket())
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java
new file mode 100644
index 0000000000000..170c80f5d4db6
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java
@@ -0,0 +1,230 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+import org.opensearch.repositories.s3.S3TransferRejectedException;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Queue implementation to accept events based on their storage attribute. If size of queue is breached, then transfer
+ * event is rejected.
+ */
+public class SizeBasedBlockingQ extends AbstractLifecycleComponent {
+    private static final Logger log = LogManager.getLogger(SizeBasedBlockingQ.class);
+
+    protected final LinkedBlockingQueue<Item> queue;
+    protected final Lock lock;
+    protected final Condition notEmpty;
+
+    protected final AtomicLong currentSize;
+    protected final ByteSizeValue capacity;
+    protected final AtomicBoolean closed;
+    protected final ExecutorService executorService;
+    protected final int consumers;
+    private final GenericStatsMetricPublisher genericStatsMetricPublisher;
+    private final QueueEventType queueEventType;
+
+    /**
+     * Constructor to create sized based blocking queue.
+     */
+    public SizeBasedBlockingQ(
+        ByteSizeValue capacity,
+        ExecutorService executorService,
+        int consumers,
+        GenericStatsMetricPublisher genericStatsMetricPublisher,
+        QueueEventType queueEventType
+    ) {
+        this.queue = new LinkedBlockingQueue<>();
+        this.lock = new ReentrantLock();
+        this.notEmpty = lock.newCondition();
+        this.currentSize = new AtomicLong();
+        this.capacity = capacity;
+        this.closed = new AtomicBoolean();
+        this.executorService = executorService;
+        this.consumers = consumers;
+        this.genericStatsMetricPublisher = genericStatsMetricPublisher;
+        this.queueEventType = queueEventType;
+    }
+
+    public enum QueueEventType {
+        NORMAL,
+        LOW;
+    }
+
+    @Override
+    protected void doStart() {
+        for (int worker = 0; worker < consumers; worker++) {
+            Thread consumer = new Consumer(queue, currentSize, lock, notEmpty, closed, genericStatsMetricPublisher, queueEventType);
+            executorService.submit(consumer);
+        }
+    }
+
+    /**
+     * Add an item to the queue
+     */
+    public void produce(Item item) throws InterruptedException {
+        if (item == null || item.size <= 0) {
+            throw new IllegalStateException("Invalid item input to produce.");
+        }
+        log.debug(() -> "Transfer queue event received of size: " + item.size + ". Current queue utilisation: " + currentSize.get());
+
+        if (currentSize.get() + item.size >= capacity.getBytes()) {
+            throw new S3TransferRejectedException("S3 Transfer queue capacity reached");
+        }
+
+        final Lock lock = this.lock;
+        final AtomicLong currentSize = this.currentSize;
+        lock.lock();
+        try {
+            if (currentSize.get() + item.size >= capacity.getBytes()) {
+                throw new S3TransferRejectedException("S3 Transfer queue capacity reached");
+            }
+            if (closed.get()) {
+                throw new AlreadyClosedException("Transfer queue is already closed.");
+            }
+            queue.put(item);
+            currentSize.addAndGet(item.size);
+            notEmpty.signalAll();
+            updateStats(item.size, queueEventType, genericStatsMetricPublisher);
+        } finally {
+            lock.unlock();
+        }
+
+    }
+
+    private static void updateStats(long itemSize, QueueEventType queueEventType, GenericStatsMetricPublisher genericStatsMetricPublisher) {
+        if (queueEventType == QueueEventType.NORMAL) {
+            genericStatsMetricPublisher.updateNormalPriorityQSize(itemSize);
+        } else if (queueEventType == QueueEventType.LOW) {
+            genericStatsMetricPublisher.updateLowPriorityQSize(itemSize);
+        }
+    }
+
+    public int getSize() {
+        return queue.size();
+    }
+
+    public boolean isMaxCapacityBelowContentLength(long contentLength) {
+        return contentLength < capacity.getBytes();
+    }
+
+    protected static class Consumer extends Thread {
+        private final LinkedBlockingQueue<Item> queue;
+        private final Lock lock;
+        private final Condition notEmpty;
+        private final AtomicLong currentSize;
+        private final AtomicBoolean closed;
+        private final GenericStatsMetricPublisher genericStatsMetricPublisher;
+        private final QueueEventType queueEventType;
+
+        public Consumer(
+            LinkedBlockingQueue<Item> queue,
+            AtomicLong currentSize,
+            Lock lock,
+            Condition notEmpty,
+            AtomicBoolean closed,
+            GenericStatsMetricPublisher genericStatsMetricPublisher,
+            QueueEventType queueEventType
+        ) {
+            this.queue = queue;
+            this.lock = lock;
+            this.notEmpty = notEmpty;
+            this.currentSize = currentSize;
+            this.closed = closed;
+            this.genericStatsMetricPublisher = genericStatsMetricPublisher;
+            this.queueEventType = queueEventType;
+        }
+
+        @Override
+        public void run() {
+            while (true) {
+                try {
+                    consume();
+                } catch (AlreadyClosedException ex) {
+                    return;
+                } catch (Exception ex) {
+                    log.error("Failed to consume transfer event", ex);
+                }
+            }
+        }
+
+        private void consume() throws InterruptedException {
+            final Lock lock = this.lock;
+            final AtomicLong currentSize = this.currentSize;
+            lock.lock();
+            Item item;
+            try {
+                if (closed.get()) {
+                    throw new AlreadyClosedException("transfer queue closed");
+                }
+                while (currentSize.get() == 0) {
+                    notEmpty.await();
+                    if (closed.get()) {
+                        throw new AlreadyClosedException("transfer queue closed");
+                    }
+                }
+
+                item = queue.take();
+                currentSize.addAndGet(-item.size);
+                updateStats(-item.size, queueEventType, genericStatsMetricPublisher);
+            } finally {
+                lock.unlock();
+            }
+
+            try {
+                item.consumable.run();
+            } catch (Exception ex) {
+                log.error("Exception on executing item consumable", ex);
+            }
+        }
+
+    }
+
+    public static class Item {
+        private final long size;
+        private final Runnable consumable;
+
+        public Item(long size, Runnable consumable) {
+            this.size = size;
+            this.consumable = consumable;
+        }
+    }
+
+    @Override
+    protected void doStop() {
+        doClose();
+    }
+
+    @Override
+    protected void doClose() {
+        lock.lock();
+        try {
+            if (closed.get() == true) {
+                return;
+            }
+            closed.set(true);
+            notEmpty.signalAll();
+        } finally {
+            lock.unlock();
+        }
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java
new file mode 100644
index 0000000000000..7dccedb8d5278
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java
@@ -0,0 +1,186 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.blobstore.stream.write.WritePriority;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+
+import java.util.Objects;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * Transfer semaphore holder for controlled transfer of data to remote.
+ */
+public class TransferSemaphoresHolder {
+    private static final Logger log = LogManager.getLogger(TransferSemaphoresHolder.class);
+    // For tests
+    protected TypeSemaphore lowPrioritySemaphore;
+    protected TypeSemaphore normalPrioritySemaphore;
+    private final int normalPriorityPermits;
+    private final int lowPriorityPermits;
+    private final int acquireWaitDuration;
+    private final TimeUnit acquireWaitDurationUnit;
+
+    /**
+     * Constructor to create semaphores holder.
+     */
+    public TransferSemaphoresHolder(
+        int normalPriorityPermits,
+        int lowPriorityPermits,
+        int acquireWaitDuration,
+        TimeUnit timeUnit,
+        GenericStatsMetricPublisher genericStatsPublisher
+    ) {
+
+        this.normalPriorityPermits = normalPriorityPermits;
+        this.lowPriorityPermits = lowPriorityPermits;
+        this.normalPrioritySemaphore = new TypeSemaphore(
+            normalPriorityPermits,
+            TypeSemaphore.PermitType.NORMAL,
+            genericStatsPublisher::updateNormalPermits
+        );
+        this.lowPrioritySemaphore = new TypeSemaphore(
+            lowPriorityPermits,
+            TypeSemaphore.PermitType.LOW,
+            genericStatsPublisher::updateLowPermits
+        );
+        this.acquireWaitDuration = acquireWaitDuration;
+        this.acquireWaitDurationUnit = timeUnit;
+    }
+
+    /**
+     * Overridden semaphore to identify transfer semaphores among all other semaphores for triaging.
+     */
+    public static class TypeSemaphore extends Semaphore {
+        private final PermitType permitType;
+        private final Consumer<Boolean> permitChangeConsumer;
+
+        public enum PermitType {
+            NORMAL,
+            LOW;
+        }
+
+        public TypeSemaphore(int permits, PermitType permitType, Consumer<Boolean> permitChangeConsumer) {
+            super(permits);
+            this.permitType = permitType;
+            this.permitChangeConsumer = permitChangeConsumer;
+        }
+
+        public PermitType getType() {
+            return permitType;
+        }
+
+        @Override
+        public boolean tryAcquire() {
+            boolean acquired = super.tryAcquire();
+            if (acquired) {
+                permitChangeConsumer.accept(true);
+            }
+            return acquired;
+        }
+
+        @Override
+        public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException {
+            boolean acquired = super.tryAcquire(timeout, unit);
+            if (acquired) {
+                permitChangeConsumer.accept(true);
+            }
+            return acquired;
+        }
+
+        @Override
+        public void release() {
+            super.release();
+            permitChangeConsumer.accept(false);
+        }
+    }
+
+    /**
+     * For multiple part requests of a single file, request context object will be set with the decision if low
+     * priority permits can also be utilized in high priority transfers of parts of the file. If high priority get fully
+     * consumed then low priority permits will be acquired for transfer.
+     *
+     * If a low priority transfer request comes in and a high priority transfer is in progress then till current
+     * high priority transfer finishes, low priority transfer may have to compete. This is an acceptable side effect
+     * because low priority transfers are generally heavy and it is ok to have slow progress in the beginning.
+     *
+     */
+    public static class RequestContext {
+
+        private final boolean lowPriorityPermitsConsumable;
+
+        private RequestContext(boolean lowPriorityPermitsConsumable) {
+            this.lowPriorityPermitsConsumable = lowPriorityPermitsConsumable;
+        }
+
+    }
+
+    public RequestContext createRequestContext() {
+        return new RequestContext(this.lowPrioritySemaphore.availablePermits() == lowPriorityPermits);
+    }
+
+    /**
+     * Acquire permit based on the availability and based on the transfer priority.
+     * A high priority event can acquire a low priority semaphore if all low permits are available.
+     * A low priority event can acquire a high priority semaphore if at least 40% of high permits are available. We
+     * reserve this bandwidth to ensure that high priority events never wait for permits in case of ongoing low priority
+     * transfers.
+     */
+    public TypeSemaphore acquirePermit(WritePriority writePriority, RequestContext requestContext) throws InterruptedException {
+        log.debug(
+            () -> "Acquire permit request for transfer type: "
+                + writePriority
+                + ". Available high priority permits: "
+                + normalPrioritySemaphore.availablePermits()
+                + " and low priority permits: "
+                + lowPrioritySemaphore.availablePermits()
+        );
+        // Try acquiring low priority permit or high priority permit immediately if available.
+        // Otherwise, we wait for low priority permit.
+        if (Objects.requireNonNull(writePriority) == WritePriority.LOW) {
+            if (lowPrioritySemaphore.tryAcquire()) {
+                return lowPrioritySemaphore;
+            } else if (normalPrioritySemaphore.availablePermits() > 0.4 * normalPriorityPermits && normalPrioritySemaphore.tryAcquire()) {
+                return normalPrioritySemaphore;
+            } else if (lowPrioritySemaphore.tryAcquire(acquireWaitDuration, acquireWaitDurationUnit)) {
+                return lowPrioritySemaphore;
+            }
+            return null;
+        }
+
+        // Try acquiring high priority permit or low priority permit immediately if available.
+        // Otherwise, we wait for high priority permit.
+        if (normalPrioritySemaphore.tryAcquire()) {
+            return normalPrioritySemaphore;
+        } else if (requestContext.lowPriorityPermitsConsumable && lowPrioritySemaphore.tryAcquire()) {
+            return lowPrioritySemaphore;
+        } else if (normalPrioritySemaphore.tryAcquire(acquireWaitDuration, acquireWaitDurationUnit)) {
+            return normalPrioritySemaphore;
+        }
+        return null;
+    }
+
+    /**
+     * Used in tests.
+     */
+    public int getNormalPriorityPermits() {
+        return normalPriorityPermits;
+    }
+
+    /**
+     * Used in tests.
+     */
+    public int getLowPriorityPermits() {
+        return lowPriorityPermits;
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java
index b944a72225d36..79b58ff215c54 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java
@@ -26,8 +26,8 @@ public class UploadRequest {
     private final CheckedConsumer<Boolean, IOException> uploadFinalizer;
     private final boolean doRemoteDataIntegrityCheck;
     private final Long expectedChecksum;
-    private boolean uploadRetryEnabled;
     private final Map<String, String> metadata;
+    private final boolean uploadRetryEnabled;
 
     /**
      * Construct a new UploadRequest object
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
index f84d953baae8e..573a4f3f51a41 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
@@ -303,7 +303,22 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, null, false) {
+            return new S3Repository(
+                metadata,
+                registry,
+                service,
+                clusterService,
+                recoverySettings,
+                null,
+                null,
+                null,
+                null,
+                null,
+                false,
+                null,
+                null,
+                null
+            ) {
                 @Override
                 protected void assertSnapshotOrGenericThread() {
                     // eliminate thread name check as we create repo manually on test/main threads
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
index 4173f8b66387f..9b413ac81d766 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
@@ -47,7 +47,10 @@
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
+import org.opensearch.repositories.s3.async.TransferSemaphoresHolder;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.Scheduler;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -65,6 +68,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -91,8 +95,13 @@ public class S3BlobContainerMockClientTests extends OpenSearchTestCase implement
     private MockS3AsyncService asyncService;
     private ExecutorService futureCompletionService;
     private ExecutorService streamReaderService;
+    private ExecutorService remoteTransferRetry;
+    private ExecutorService transferQueueConsumerService;
+    private ScheduledExecutorService scheduler;
     private AsyncTransferEventLoopGroup transferNIOGroup;
     private S3BlobContainer blobContainer;
+    private SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
 
     static class MockS3AsyncService extends S3AsyncService {
 
@@ -364,7 +373,27 @@ public void setUp() throws Exception {
         asyncService = new MockS3AsyncService(configPath(), 1000);
         futureCompletionService = Executors.newSingleThreadExecutor();
         streamReaderService = Executors.newSingleThreadExecutor();
+        remoteTransferRetry = Executors.newFixedThreadPool(20);
+        transferQueueConsumerService = Executors.newFixedThreadPool(20);
+        scheduler = new Scheduler.SafeScheduledThreadPoolExecutor(1);
         transferNIOGroup = new AsyncTransferEventLoopGroup(1);
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        normalPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 10L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            10,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+        lowPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 20L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            5,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+        normalPrioritySizeBasedBlockingQ.start();
+        lowPrioritySizeBasedBlockingQ.start();
         blobContainer = createBlobContainer();
         super.setUp();
     }
@@ -373,6 +402,14 @@ public void setUp() throws Exception {
     @After
     public void tearDown() throws Exception {
         IOUtils.close(asyncService);
+        futureCompletionService.shutdown();
+        streamReaderService.shutdown();
+        remoteTransferRetry.shutdown();
+        transferQueueConsumerService.shutdown();
+        normalPrioritySizeBasedBlockingQ.close();
+        lowPrioritySizeBasedBlockingQ.close();
+        scheduler.shutdown();
+        transferNIOGroup.close();
         super.tearDown();
     }
 
@@ -394,7 +431,7 @@ private S3BlobStore createBlobStore() {
             streamReaderService,
             transferNIOGroup
         );
-
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
         return new S3BlobStore(
             null,
             asyncService,
@@ -410,11 +447,21 @@ private S3BlobStore createBlobStore() {
                 S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
                 asyncExecutorContainer.getStreamReader(),
                 asyncExecutorContainer.getStreamReader(),
-                asyncExecutorContainer.getStreamReader()
+                asyncExecutorContainer.getStreamReader(),
+                new TransferSemaphoresHolder(
+                    3,
+                    Math.max(Runtime.getRuntime().availableProcessors() * 5, 10),
+                    5,
+                    TimeUnit.MINUTES,
+                    genericStatsMetricPublisher
+                )
             ),
             asyncExecutorContainer,
             asyncExecutorContainer,
-            asyncExecutorContainer
+            asyncExecutorContainer,
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
@@ -574,19 +621,32 @@ private int calculateNumberOfParts(long contentLength, long partSize) {
         return (int) ((contentLength % partSize) == 0 ? contentLength / partSize : (contentLength / partSize) + 1);
     }
 
-    public void testFailureWhenLargeFileRedirected() throws IOException, ExecutionException, InterruptedException {
-        testLargeFilesRedirectedToSlowSyncClient(true);
+    public void testFailureWhenLargeFileRedirected() throws IOException, InterruptedException {
+        testLargeFilesRedirectedToSlowSyncClient(true, WritePriority.LOW);
+        testLargeFilesRedirectedToSlowSyncClient(true, WritePriority.NORMAL);
     }
 
-    public void testLargeFileRedirected() throws IOException, ExecutionException, InterruptedException {
-        testLargeFilesRedirectedToSlowSyncClient(false);
+    public void testLargeFileRedirected() throws IOException, InterruptedException {
+        testLargeFilesRedirectedToSlowSyncClient(false, WritePriority.LOW);
+        testLargeFilesRedirectedToSlowSyncClient(false, WritePriority.NORMAL);
     }
 
-    private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) throws IOException, InterruptedException {
-        final ByteSizeValue partSize = new ByteSizeValue(1024, ByteSizeUnit.MB);
-
+    private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException, WritePriority writePriority) throws IOException,
+        InterruptedException {
+        ByteSizeValue capacity = new ByteSizeValue(1, ByteSizeUnit.GB);
         int numberOfParts = 20;
-        final long lastPartSize = new ByteSizeValue(20, ByteSizeUnit.MB).getBytes();
+        final ByteSizeValue partSize = new ByteSizeValue(capacity.getBytes() / numberOfParts + 1, ByteSizeUnit.BYTES);
+
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        SizeBasedBlockingQ sizeBasedBlockingQ = new SizeBasedBlockingQ(
+            capacity,
+            transferQueueConsumerService,
+            10,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+
+        final long lastPartSize = new ByteSizeValue(200, ByteSizeUnit.MB).getBytes();
         final long blobSize = ((numberOfParts - 1) * partSize.getBytes()) + lastPartSize;
         CountDownLatch countDownLatch = new CountDownLatch(1);
         AtomicReference<Exception> exceptionRef = new AtomicReference<>();
@@ -609,6 +669,9 @@ private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) t
         when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
         when(blobStore.bufferSizeInBytes()).thenReturn(bufferSize);
 
+        when(blobStore.getLowPrioritySizeBasedBlockingQ()).thenReturn(sizeBasedBlockingQ);
+        when(blobStore.getNormalPrioritySizeBasedBlockingQ()).thenReturn(sizeBasedBlockingQ);
+
         final boolean serverSideEncryption = randomBoolean();
         when(blobStore.serverSideEncryption()).thenReturn(serverSideEncryption);
 
@@ -658,7 +721,7 @@ private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) t
             .streamContextSupplier(streamContextSupplier)
             .fileSize(blobSize)
             .failIfAlreadyExists(false)
-            .writePriority(WritePriority.HIGH)
+            .writePriority(writePriority)
             .uploadFinalizer(Assert::assertTrue)
             .doRemoteDataIntegrityCheck(false)
             .metadata(new HashMap<>())
@@ -693,5 +756,4 @@ private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) t
             }
         });
     }
-
 }
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
index 10578090da75c..96ef28d24c14f 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -67,6 +67,8 @@
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
+import org.opensearch.repositories.s3.async.TransferSemaphoresHolder;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -87,6 +89,8 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -114,7 +118,12 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes
     private S3AsyncService asyncService;
     private ExecutorService futureCompletionService;
     private ExecutorService streamReaderService;
+    private ExecutorService remoteTransferRetry;
+    private ExecutorService transferQueueConsumerService;
+    private ScheduledExecutorService scheduler;
     private AsyncTransferEventLoopGroup transferNIOGroup;
+    private SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
 
     @Before
     public void setUp() throws Exception {
@@ -125,7 +134,26 @@ public void setUp() throws Exception {
         futureCompletionService = Executors.newSingleThreadExecutor();
         streamReaderService = Executors.newSingleThreadExecutor();
         transferNIOGroup = new AsyncTransferEventLoopGroup(1);
-
+        remoteTransferRetry = Executors.newFixedThreadPool(20);
+        transferQueueConsumerService = Executors.newFixedThreadPool(2);
+        scheduler = new ScheduledThreadPoolExecutor(1);
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        normalPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 5L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            2,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+        lowPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 5L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            2,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.LOW
+        );
+        normalPrioritySizeBasedBlockingQ.start();
+        lowPrioritySizeBasedBlockingQ.start();
         // needed by S3AsyncService
         SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", configPath().toString()));
         super.setUp();
@@ -137,6 +165,11 @@ public void tearDown() throws Exception {
 
         streamReaderService.shutdown();
         futureCompletionService.shutdown();
+        remoteTransferRetry.shutdown();
+        transferQueueConsumerService.shutdown();
+        scheduler.shutdown();
+        normalPrioritySizeBasedBlockingQ.close();
+        lowPrioritySizeBasedBlockingQ.close();
         IOUtils.close(transferNIOGroup);
 
         if (previousOpenSearchPathConf != null) {
@@ -205,7 +238,7 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
             streamReaderService,
             transferNIOGroup
         );
-
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
         return new S3BlobContainer(
             BlobPath.cleanPath(),
             new S3BlobStore(
@@ -223,11 +256,21 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
                     S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
                     asyncExecutorContainer.getStreamReader(),
                     asyncExecutorContainer.getStreamReader(),
-                    asyncExecutorContainer.getStreamReader()
+                    asyncExecutorContainer.getStreamReader(),
+                    new TransferSemaphoresHolder(
+                        3,
+                        Math.max(Runtime.getRuntime().availableProcessors() * 5, 10),
+                        5,
+                        TimeUnit.MINUTES,
+                        genericStatsMetricPublisher
+                    )
                 ),
                 asyncExecutorContainer,
                 asyncExecutorContainer,
-                asyncExecutorContainer
+                asyncExecutorContainer,
+                normalPrioritySizeBasedBlockingQ,
+                lowPrioritySizeBasedBlockingQ,
+                genericStatsMetricPublisher
             )
         ) {
             @Override
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
index 6fec535ae6301..f8e9903bb3577 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
@@ -169,7 +169,11 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) {
             null,
             null,
             null,
-            false
+            false,
+            null,
+            null,
+            null,
+            null
         ) {
             @Override
             protected void assertSnapshotOrGenericThread() {
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
index 04d1819bef02b..89add3cdbfc60 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
@@ -33,6 +33,7 @@
 import org.opensearch.common.io.InputStreamContainer;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.blobstore.ZeroInputStream;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
 import org.opensearch.repositories.s3.StatsMetricPublisher;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
@@ -46,6 +47,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.mockito.ArgumentMatchers.any;
@@ -63,11 +65,19 @@ public class AsyncTransferManagerTests extends OpenSearchTestCase {
     @Before
     public void setUp() throws Exception {
         s3AsyncClient = mock(S3AsyncClient.class);
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
         asyncTransferManager = new AsyncTransferManager(
             ByteSizeUnit.MB.toBytes(5),
             Executors.newSingleThreadExecutor(),
             Executors.newSingleThreadExecutor(),
-            Executors.newSingleThreadExecutor()
+            Executors.newSingleThreadExecutor(),
+            new TransferSemaphoresHolder(
+                3,
+                Math.max(Runtime.getRuntime().availableProcessors() * 5, 10),
+                5,
+                TimeUnit.MINUTES,
+                genericStatsMetricPublisher
+            )
         );
         super.setUp();
     }
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java
new file mode 100644
index 0000000000000..5be4037407d23
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java
@@ -0,0 +1,102 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+import org.opensearch.repositories.s3.S3TransferRejectedException;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class SizeBasedBlockingQTests extends OpenSearchTestCase {
+    private ExecutorService consumerService;
+    private ExecutorService producerService;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        this.consumerService = Executors.newFixedThreadPool(10);
+        this.producerService = Executors.newFixedThreadPool(100);
+        super.setUp();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        consumerService.shutdown();
+        producerService.shutdown();
+        super.tearDown();
+    }
+
+    public void testProducerConsumerOfBulkItems() throws InterruptedException {
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        SizeBasedBlockingQ.QueueEventType queueEventType = randomBoolean()
+            ? SizeBasedBlockingQ.QueueEventType.NORMAL
+            : SizeBasedBlockingQ.QueueEventType.LOW;
+        SizeBasedBlockingQ sizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(ByteSizeUnit.BYTES.toBytes(10)),
+            consumerService,
+            10,
+            genericStatsMetricPublisher,
+            queueEventType
+        );
+        sizeBasedBlockingQ.start();
+        int numOfItems = randomIntBetween(100, 1000);
+        CountDownLatch latch = new CountDownLatch(numOfItems);
+        AtomicBoolean unknownError = new AtomicBoolean();
+        for (int i = 0; i < numOfItems; i++) {
+            final int idx = i;
+            producerService.submit(() -> {
+                boolean throwException = randomBoolean();
+
+                SizeBasedBlockingQ.Item item = new TestItemToStr(randomIntBetween(1, 5), () -> {
+                    latch.countDown();
+                    if (throwException) {
+                        throw new RuntimeException("throwing random exception");
+                    }
+                }, idx);
+
+                try {
+                    sizeBasedBlockingQ.produce(item);
+                } catch (InterruptedException e) {
+                    latch.countDown();
+                    unknownError.set(true);
+                    throw new RuntimeException(e);
+                } catch (S3TransferRejectedException ex) {
+                    latch.countDown();
+                }
+            });
+        }
+        latch.await();
+        sizeBasedBlockingQ.close();
+        assertFalse(unknownError.get());
+        assertEquals(0L, genericStatsMetricPublisher.getNormalPriorityQSize());
+        assertEquals(0L, genericStatsMetricPublisher.getLowPriorityQSize());
+    }
+
+    static class TestItemToStr extends SizeBasedBlockingQ.Item {
+        private final int id;
+
+        public TestItemToStr(long size, Runnable consumable, int id) {
+            super(size, consumable);
+            this.id = id;
+        }
+
+        @Override
+        public String toString() {
+            return String.valueOf(id);
+        }
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java
new file mode 100644
index 0000000000000..236f02c5eb1f7
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java
@@ -0,0 +1,276 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.opensearch.common.blobstore.stream.write.WritePriority;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.mockito.Mockito;
+
+import static org.opensearch.repositories.s3.async.TransferSemaphoresHolder.TypeSemaphore.PermitType;
+
+public class TransferSemaphoresHolderTests extends OpenSearchTestCase {
+
+    public void testAllocation() {
+        int availablePermits = randomIntBetween(5, 20);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TransferSemaphoresHolder transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            1,
+            TimeUnit.NANOSECONDS,
+            genericStatsPublisher
+        );
+        assertEquals(normalPermits, transferSemaphoresHolder.getNormalPriorityPermits());
+        assertEquals(lowPermits, transferSemaphoresHolder.getLowPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredNormalPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredLowPriorityPermits());
+    }
+
+    public void testLowPriorityEventPermitAcquisition() throws InterruptedException {
+        int availablePermits = randomIntBetween(5, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TransferSemaphoresHolder transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            1,
+            TimeUnit.NANOSECONDS,
+            genericStatsPublisher
+        );
+
+        List<TransferSemaphoresHolder.TypeSemaphore> semaphores = new ArrayList<>();
+        int normalPermitsEligibleForLowEvents = normalPermits - (int) (normalPermits * 0.4);
+
+        int lowAcquisitionsExpected = (normalPermitsEligibleForLowEvents + lowPermits);
+        for (int i = 0; i < lowAcquisitionsExpected; i++) {
+            TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.LOW,
+                requestContext
+            );
+            semaphores.add(acquiredSemaphore);
+            if (i >= lowPermits) {
+                assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+            } else {
+                assertEquals(PermitType.LOW, acquiredSemaphore.getType());
+            }
+        }
+
+        for (int i = 0; i < normalPermits - normalPermitsEligibleForLowEvents; i++) {
+            TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+            semaphores.add(acquiredSemaphore);
+        }
+
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.LOW,
+            requestContext
+        );
+        assertNull(acquiredSemaphore);
+
+        assertEquals(availablePermits, semaphores.size());
+        semaphores.forEach(Semaphore::release);
+        assertEquals(normalPermits, transferSemaphoresHolder.getNormalPriorityPermits());
+        assertEquals(lowPermits, transferSemaphoresHolder.getLowPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredNormalPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredLowPriorityPermits());
+
+    }
+
+    public void testNormalPermitEventAcquisition() throws InterruptedException {
+        int availablePermits = randomIntBetween(5, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TransferSemaphoresHolder transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            1,
+            TimeUnit.NANOSECONDS,
+            genericStatsPublisher
+        );
+
+        List<TransferSemaphoresHolder.TypeSemaphore> semaphores = new ArrayList<>();
+        List<TransferSemaphoresHolder.TypeSemaphore> lowSemaphores = new ArrayList<>();
+        int normalAcquisitionsExpected = normalPermits + lowPermits;
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        for (int i = 0; i < normalAcquisitionsExpected; i++) {
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            semaphores.add(acquiredSemaphore);
+            if (i >= normalPermits) {
+                assertEquals(PermitType.LOW, acquiredSemaphore.getType());
+                lowSemaphores.add(acquiredSemaphore);
+            } else {
+                assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+            }
+        }
+        assertEquals(availablePermits, semaphores.size());
+
+        int lowAcquired = lowPermits;
+
+        Semaphore removedLowSemaphore = lowSemaphores.remove(0);
+        removedLowSemaphore.release();
+        semaphores.remove(removedLowSemaphore);
+
+        requestContext = transferSemaphoresHolder.createRequestContext();
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.LOW,
+            requestContext
+        );
+        semaphores.add(acquiredSemaphore);
+        lowSemaphores.add(acquiredSemaphore);
+        while (lowAcquired > 1) {
+            requestContext = transferSemaphoresHolder.createRequestContext();
+            acquiredSemaphore = transferSemaphoresHolder.acquirePermit(WritePriority.NORMAL, requestContext);
+            assertNull(acquiredSemaphore);
+            lowAcquired--;
+        }
+
+        semaphores.forEach(Semaphore::release);
+        assertEquals(normalPermits, transferSemaphoresHolder.getNormalPriorityPermits());
+        assertEquals(lowPermits, transferSemaphoresHolder.getLowPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredNormalPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredLowPriorityPermits());
+    }
+
+    private static class TestTransferSemaphoresHolder extends TransferSemaphoresHolder {
+        AtomicInteger normalWaitCount = new AtomicInteger();
+        AtomicInteger lowWaitCount = new AtomicInteger();
+
+        /**
+         * Constructor to create semaphores holder.
+         */
+        public TestTransferSemaphoresHolder(
+            int normalPermits,
+            int lowPermits,
+            int acquireWaitDuration,
+            TimeUnit timeUnit,
+            GenericStatsMetricPublisher genericStatsMetricPublisher
+        ) throws InterruptedException {
+            super(normalPermits, lowPermits, acquireWaitDuration, timeUnit, genericStatsMetricPublisher);
+            TypeSemaphore executingNormalSemaphore = normalPrioritySemaphore;
+            TypeSemaphore executingLowSemaphore = lowPrioritySemaphore;
+
+            this.normalPrioritySemaphore = Mockito.spy(normalPrioritySemaphore);
+            this.lowPrioritySemaphore = Mockito.spy(lowPrioritySemaphore);
+            Mockito.doAnswer(invocation -> {
+                normalWaitCount.incrementAndGet();
+                return false;
+            }).when(normalPrioritySemaphore).tryAcquire(Mockito.anyLong(), Mockito.any(TimeUnit.class));
+            Mockito.doAnswer(invocation -> executingNormalSemaphore.availablePermits()).when(normalPrioritySemaphore).availablePermits();
+            Mockito.doAnswer(invocation -> executingNormalSemaphore.tryAcquire()).when(normalPrioritySemaphore).tryAcquire();
+
+            Mockito.doAnswer(invocation -> {
+                lowWaitCount.incrementAndGet();
+                return false;
+            }).when(lowPrioritySemaphore).tryAcquire(Mockito.anyLong(), Mockito.any(TimeUnit.class));
+            Mockito.doAnswer(invocation -> executingLowSemaphore.availablePermits()).when(lowPrioritySemaphore).availablePermits();
+            Mockito.doAnswer(invocation -> executingLowSemaphore.tryAcquire()).when(lowPrioritySemaphore).tryAcquire();
+        }
+    }
+
+    public void testNormalSemaphoreAcquiredWait() throws InterruptedException {
+        int availablePermits = randomIntBetween(10, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TestTransferSemaphoresHolder transferSemaphoresHolder = new TestTransferSemaphoresHolder(
+            normalPermits,
+            availablePermits - normalPermits,
+            5,
+            TimeUnit.MINUTES,
+            genericStatsPublisher
+        );
+
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        TransferSemaphoresHolder.TypeSemaphore lowSemaphore = transferSemaphoresHolder.acquirePermit(WritePriority.LOW, requestContext);
+        assertEquals(PermitType.LOW, lowSemaphore.getType());
+        for (int i = 0; i < normalPermits; i++) {
+            requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+        }
+
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.NORMAL,
+            requestContext
+        );
+        assertNull(acquiredSemaphore);
+        assertEquals(1, transferSemaphoresHolder.normalWaitCount.get());
+        assertEquals(0, transferSemaphoresHolder.lowWaitCount.get());
+    }
+
+    public void testLowSemaphoreAcquiredWait() throws InterruptedException {
+        int availablePermits = randomIntBetween(10, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TestTransferSemaphoresHolder transferSemaphoresHolder = new TestTransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            5,
+            TimeUnit.MINUTES,
+            genericStatsPublisher
+        );
+
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        int normalPermitsEligibleForLowEvents = normalPermits - (int) (normalPermits * 0.4);
+        for (int i = 0; i < normalPermitsEligibleForLowEvents; i++) {
+            TransferSemaphoresHolder.TypeSemaphore lowSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            assertEquals(PermitType.NORMAL, lowSemaphore.getType());
+        }
+
+        for (int i = 0; i < lowPermits; i++) {
+            requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.LOW,
+                requestContext
+            );
+            assertEquals(PermitType.LOW, acquiredSemaphore.getType());
+        }
+
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.LOW,
+            requestContext
+        );
+        assertNull(acquiredSemaphore);
+        assertEquals(1, transferSemaphoresHolder.lowWaitCount.get());
+        assertEquals(0, transferSemaphoresHolder.normalWaitCount.get());
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
index 0f6646d37f950..8ce8ec8e01abe 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
@@ -75,6 +75,7 @@ default void reload(RepositoryMetadata repositoryMetadata) {}
      * Metrics for BlobStore interactions
      */
     enum Metric {
+        GENERIC_STATS("generic_stats"),
         REQUEST_SUCCESS("request_success_total"),
         REQUEST_FAILURE("request_failures_total"),
         REQUEST_LATENCY("request_time_in_millis"),
diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
index 3f341c878c3c7..4e8db0a3a8c69 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
@@ -14,7 +14,12 @@
  * @opensearch.internal
  */
 public enum WritePriority {
+    // Used for segment transfers during refresh, flush or merges
     NORMAL,
+    // Used for transfer of translog or ckp files.
     HIGH,
-    URGENT
+    // Used for transfer of remote cluster state
+    URGENT,
+    // All other background transfers such as in snapshot recovery, recovery from local store or index etc.
+    LOW
 }
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index 351aec6e3af6c..bfb841307af49 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -437,10 +437,14 @@ private void uploadNewSegments(
                 batchUploadListener.onFailure(ex);
             });
             statsListener.beforeUpload(src);
-            remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener);
+            remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener, isLowPriorityUpload());
         }
     }
 
+    private boolean isLowPriorityUpload() {
+        return isLocalOrSnapshotRecovery();
+    }
+
     /**
      * Whether to upload a file or not depending on whether file is in excluded list or has been already uploaded.
      *
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
index 345583bbbd1be..ab76150f8f83d 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
@@ -29,6 +29,7 @@
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream;
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeInputStream;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.index.store.exception.ChecksumCombinationException;
 
 import java.io.FileNotFoundException;
@@ -323,11 +324,12 @@ public boolean copyFrom(
         String remoteFileName,
         IOContext context,
         Runnable postUploadRunner,
-        ActionListener<Void> listener
+        ActionListener<Void> listener,
+        boolean lowPriorityUpload
     ) {
         if (blobContainer instanceof AsyncMultiStreamBlobContainer) {
             try {
-                uploadBlob(from, src, remoteFileName, context, postUploadRunner, listener);
+                uploadBlob(from, src, remoteFileName, context, postUploadRunner, listener, lowPriorityUpload);
             } catch (Exception e) {
                 listener.onFailure(e);
             }
@@ -342,7 +344,8 @@ private void uploadBlob(
         String remoteFileName,
         IOContext ioContext,
         Runnable postUploadRunner,
-        ActionListener<Void> listener
+        ActionListener<Void> listener,
+        boolean lowPriorityUpload
     ) throws Exception {
         long expectedChecksum = calculateChecksumOfChecksum(from, src);
         long contentLength;
@@ -353,12 +356,13 @@ private void uploadBlob(
         if (getBlobContainer() instanceof AsyncMultiStreamBlobContainer) {
             remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) getBlobContainer()).remoteIntegrityCheckSupported();
         }
+        lowPriorityUpload = lowPriorityUpload || contentLength > ByteSizeUnit.GB.toBytes(15);
         RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer(
             src,
             remoteFileName,
             contentLength,
             true,
-            WritePriority.NORMAL,
+            lowPriorityUpload ? WritePriority.LOW : WritePriority.NORMAL,
             (size, position) -> uploadRateLimiter.apply(new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position)),
             expectedChecksum,
             remoteIntegrityEnabled
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
index ec1163fe91b6c..8c0ecb4cc783a 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
@@ -453,7 +453,7 @@ public IndexInput openInput(String name, IOContext context) throws IOException {
      * @param context  IOContext to be used to open IndexInput of file during remote upload
      * @param listener Listener to handle upload callback events
      */
-    public void copyFrom(Directory from, String src, IOContext context, ActionListener<Void> listener) {
+    public void copyFrom(Directory from, String src, IOContext context, ActionListener<Void> listener, boolean lowPriorityUpload) {
         try {
             final String remoteFileName = getNewRemoteSegmentFilename(src);
             boolean uploaded = remoteDataDirectory.copyFrom(from, src, remoteFileName, context, () -> {
@@ -462,7 +462,7 @@ public void copyFrom(Directory from, String src, IOContext context, ActionListen
                 } catch (IOException e) {
                     throw new RuntimeException("Exception in segment postUpload for file " + src, e);
                 }
-            }, listener);
+            }, listener, lowPriorityUpload);
             if (uploaded == false) {
                 copyFrom(from, src, src, context);
                 listener.onResponse(null);
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
index 9e38e1749d434..ee81369725e6f 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
@@ -104,7 +104,8 @@ public void onResponse(Void t) {
                 public void onFailure(Exception e) {
                     fail("Listener responded with exception" + e);
                 }
-            }
+            },
+            false
         );
         assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
         assertTrue(postUploadInvoked.get());
@@ -141,7 +142,8 @@ public void onResponse(Void t) {
                 public void onFailure(Exception e) {
                     countDownLatch.countDown();
                 }
-            }
+            },
+            false
         );
         assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
         assertFalse(postUploadInvoked.get());
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
index b1e2028d761f0..567199cf64cd8 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
@@ -639,7 +639,7 @@ public void onResponse(Void unused) {
             @Override
             public void onFailure(Exception e) {}
         };
-        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener);
+        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener, false);
         assertTrue(latch.await(5000, TimeUnit.SECONDS));
         assertTrue(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename));
         storeDirectory.close();
@@ -683,7 +683,7 @@ public void onFailure(Exception e) {
                 latch.countDown();
             }
         };
-        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener);
+        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener, false);
         assertTrue(latch.await(5000, TimeUnit.SECONDS));
         assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename));
 

From a07858fd0f7e563ed2e4c17934dbbe37f6557f96 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 10:44:50 -0400
Subject: [PATCH 30/52] Bump com.gradle.enterprise from 3.17.2 to 3.17.3
 (#13641)

* Bump com.gradle.enterprise from 3.17.2 to 3.17.3

Bumps com.gradle.enterprise from 3.17.2 to 3.17.3.

---
updated-dependencies:
- dependency-name: com.gradle.enterprise
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md    | 1 +
 settings.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b24cc5347e768..f5d0ab4f7af38 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
 - Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
+- Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
diff --git a/settings.gradle b/settings.gradle
index 8e961b9d4179f..065a8dc3d0a8a 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -10,7 +10,7 @@
  */
 
 plugins {
-  id "com.gradle.enterprise" version "3.17.2"
+  id "com.gradle.enterprise" version "3.17.3"
 }
 
 ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE')

From 079cef5445b89b8072ad0ff811561e1d12c84603 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 14:02:45 -0700
Subject: [PATCH 31/52] Bump org.apache.hadoop:hadoop-minicluster from 3.3.6 to
 3.4.0 in /test/fixtures/hdfs-fixture (#13642)

* Bump org.apache.hadoop:hadoop-minicluster in /test/fixtures/hdfs-fixture

Bumps org.apache.hadoop:hadoop-minicluster from 3.3.6 to 3.4.0.

---
updated-dependencies:
- dependency-name: org.apache.hadoop:hadoop-minicluster
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                            | 1 +
 test/fixtures/hdfs-fixture/build.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f5d0ab4f7af38..afb0d0840b405 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
 - Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
+- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 8ac13d897e0b5..33635f2053a21 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -37,7 +37,7 @@ versions << [
 ]
 
 dependencies {
-  api("org.apache.hadoop:hadoop-minicluster:3.3.6") {
+  api("org.apache.hadoop:hadoop-minicluster:3.4.0") {
     exclude module: 'websocket-client'
     exclude module: 'jettison'
     exclude module: 'netty'

From 14f1c43c108f378b13d109ade364216c082fb858 Mon Sep 17 00:00:00 2001
From: Vacha Shah <vachshah@amazon.com>
Date: Mon, 13 May 2024 15:06:27 -0700
Subject: [PATCH 32/52] Abstracting outbound side of transport (#13293)

* Abstracting outbound side of transport

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Making outbound handler protocol dependent via inbound handler

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Fixing precommit

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Addressing comments

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Fixing precommit

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Cleaning up code

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Addressing comments

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Cleaning up

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Addressing comments

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Abstracting InboundHandlerTests

Signed-off-by: Vacha Shah <vachshah@amazon.com>

* Abstracting TransportLoggerTests

Signed-off-by: Vacha Shah <vachshah@amazon.com>

---------

Signed-off-by: Vacha Shah <vachshah@amazon.com>
---
 CHANGELOG.md                                  |   1 +
 .../opensearch/transport/InboundHandler.java  |  13 +
 .../transport/NativeMessageHandler.java       |  16 +-
 .../opensearch/transport/OutboundHandler.java | 171 +---------
 .../transport/ProtocolMessageHandler.java     |  14 +
 .../transport/ProtocolOutboundHandler.java    |  70 ++++
 .../opensearch/transport/TcpTransport.java    |  24 +-
 .../transport/TcpTransportChannel.java        |   4 +-
 .../opensearch/transport/TransportStatus.java |   4 +-
 .../CompressibleBytesOutputStream.java        |   2 +-
 .../nativeprotocol/NativeOutboundHandler.java | 224 +++++++++++++
 .../NativeOutboundMessage.java}               |  18 +-
 .../{ => nativeprotocol}/NetworkMessage.java  |   3 +-
 .../transport/InboundDecoderTests.java        | 108 +++----
 .../transport/InboundHandlerTests.java        |  89 +++---
 .../transport/InboundPipelineTests.java       | 111 ++-----
 .../transport/NativeOutboundHandlerTests.java | 302 ++++++++++++++++++
 .../transport/OutboundHandlerTests.java       | 226 +------------
 .../transport/TcpTransportTests.java          |  15 +-
 .../transport/TransportLoggerTests.java       |  25 +-
 .../CompressibleBytesOutputStreamTests.java   |   2 +-
 .../NativeInboundDecoderTests.java            |  59 ++++
 .../NativeInboundHandlerTests.java            |  45 +++
 .../NativeInboundPipelineTests.java           |  60 ++++
 .../NativeOutboundMessageTests.java           |  50 +++
 .../NativeTransportLoggerTests.java           |  42 +++
 .../org/opensearch/transport/TestRequest.java |   4 +
 .../opensearch/transport/TestResponse.java    |   4 +
 28 files changed, 1061 insertions(+), 645 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java
 rename server/src/main/java/org/opensearch/transport/{ => nativeprotocol}/CompressibleBytesOutputStream.java (98%)
 create mode 100644 server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java
 rename server/src/main/java/org/opensearch/transport/{OutboundMessage.java => nativeprotocol/NativeOutboundMessage.java} (91%)
 rename server/src/main/java/org/opensearch/transport/{ => nativeprotocol}/NetworkMessage.java (96%)
 create mode 100644 server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java
 rename server/src/test/java/org/opensearch/transport/{ => nativeprotocol}/CompressibleBytesOutputStreamTests.java (99%)
 create mode 100644 server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java
 create mode 100644 server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java
 create mode 100644 server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java
 create mode 100644 server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java
 create mode 100644 server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index afb0d0840b405..b858ede4b78cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
+- Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
 
 ### Dependencies
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
diff --git a/server/src/main/java/org/opensearch/transport/InboundHandler.java b/server/src/main/java/org/opensearch/transport/InboundHandler.java
index 6492900c49a0e..f77c44ea362cf 100644
--- a/server/src/main/java/org/opensearch/transport/InboundHandler.java
+++ b/server/src/main/java/org/opensearch/transport/InboundHandler.java
@@ -32,7 +32,9 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.Version;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.BigArrays;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
@@ -57,7 +59,12 @@ public class InboundHandler {
     private final Map<String, ProtocolMessageHandler> protocolMessageHandlers;
 
     InboundHandler(
+        String nodeName,
+        Version version,
+        String[] features,
+        StatsTracker statsTracker,
         ThreadPool threadPool,
+        BigArrays bigArrays,
         OutboundHandler outboundHandler,
         NamedWriteableRegistry namedWriteableRegistry,
         TransportHandshaker handshaker,
@@ -70,7 +77,12 @@ public class InboundHandler {
         this.protocolMessageHandlers = Map.of(
             NativeInboundMessage.NATIVE_PROTOCOL,
             new NativeMessageHandler(
+                nodeName,
+                version,
+                features,
+                statsTracker,
                 threadPool,
+                bigArrays,
                 outboundHandler,
                 namedWriteableRegistry,
                 handshaker,
@@ -83,6 +95,7 @@ public class InboundHandler {
     }
 
     void setMessageListener(TransportMessageListener listener) {
+        protocolMessageHandlers.values().forEach(handler -> handler.setMessageListener(listener));
         if (messageListener == TransportMessageListener.NOOP_LISTENER) {
             messageListener = listener;
         } else {
diff --git a/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java b/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java
index c5b65f9eb7a11..4c972fdc14fa5 100644
--- a/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java
+++ b/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java
@@ -37,6 +37,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.Version;
+import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.concurrent.AbstractRunnable;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.io.stream.ByteBufferStreamInput;
@@ -52,6 +53,7 @@
 import org.opensearch.telemetry.tracing.channels.TraceableTcpTransportChannel;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
+import org.opensearch.transport.nativeprotocol.NativeOutboundHandler;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -72,7 +74,7 @@ public class NativeMessageHandler implements ProtocolMessageHandler {
     private static final Logger logger = LogManager.getLogger(NativeMessageHandler.class);
 
     private final ThreadPool threadPool;
-    private final OutboundHandler outboundHandler;
+    private final NativeOutboundHandler outboundHandler;
     private final NamedWriteableRegistry namedWriteableRegistry;
     private final TransportHandshaker handshaker;
     private final TransportKeepAlive keepAlive;
@@ -82,7 +84,12 @@ public class NativeMessageHandler implements ProtocolMessageHandler {
     private final Tracer tracer;
 
     NativeMessageHandler(
+        String nodeName,
+        Version version,
+        String[] features,
+        StatsTracker statsTracker,
         ThreadPool threadPool,
+        BigArrays bigArrays,
         OutboundHandler outboundHandler,
         NamedWriteableRegistry namedWriteableRegistry,
         TransportHandshaker handshaker,
@@ -92,7 +99,7 @@ public class NativeMessageHandler implements ProtocolMessageHandler {
         TransportKeepAlive keepAlive
     ) {
         this.threadPool = threadPool;
-        this.outboundHandler = outboundHandler;
+        this.outboundHandler = new NativeOutboundHandler(nodeName, version, features, statsTracker, threadPool, bigArrays, outboundHandler);
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.handshaker = handshaker;
         this.requestHandlers = requestHandlers;
@@ -492,4 +499,9 @@ public void onFailure(Exception e) {
         }
     }
 
+    @Override
+    public void setMessageListener(TransportMessageListener listener) {
+        outboundHandler.setMessageListener(listener);
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java
index b83dbdd0effe4..43f53e4011260 100644
--- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java
+++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java
@@ -35,164 +35,47 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.opensearch.Version;
-import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.CheckedSupplier;
-import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.common.transport.NetworkExceptionHelper;
-import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.NotifyOnceListener;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.transport.TransportAddress;
-import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.util.Set;
 
 /**
  * Outbound data handler
  *
  * @opensearch.internal
  */
-final class OutboundHandler {
+public final class OutboundHandler {
 
     private static final Logger logger = LogManager.getLogger(OutboundHandler.class);
 
-    private final String nodeName;
-    private final Version version;
-    private final String[] features;
     private final StatsTracker statsTracker;
     private final ThreadPool threadPool;
-    private final BigArrays bigArrays;
-    private volatile TransportMessageListener messageListener = TransportMessageListener.NOOP_LISTENER;
 
-    OutboundHandler(
-        String nodeName,
-        Version version,
-        String[] features,
-        StatsTracker statsTracker,
-        ThreadPool threadPool,
-        BigArrays bigArrays
-    ) {
-        this.nodeName = nodeName;
-        this.version = version;
-        this.features = features;
+    public OutboundHandler(StatsTracker statsTracker, ThreadPool threadPool) {
         this.statsTracker = statsTracker;
         this.threadPool = threadPool;
-        this.bigArrays = bigArrays;
     }
 
     void sendBytes(TcpChannel channel, BytesReference bytes, ActionListener<Void> listener) {
-        SendContext sendContext = new SendContext(channel, () -> bytes, listener);
+        SendContext sendContext = new SendContext(statsTracker, channel, () -> bytes, listener);
         try {
-            internalSend(channel, sendContext);
+            sendBytes(channel, sendContext);
         } catch (IOException e) {
             // This should not happen as the bytes are already serialized
             throw new AssertionError(e);
         }
     }
 
-    /**
-     * Sends the request to the given channel. This method should be used to send {@link TransportRequest}
-     * objects back to the caller.
-     */
-    void sendRequest(
-        final DiscoveryNode node,
-        final TcpChannel channel,
-        final long requestId,
-        final String action,
-        final TransportRequest request,
-        final TransportRequestOptions options,
-        final Version channelVersion,
-        final boolean compressRequest,
-        final boolean isHandshake
-    ) throws IOException, TransportException {
-        Version version = Version.min(this.version, channelVersion);
-        OutboundMessage.Request message = new OutboundMessage.Request(
-            threadPool.getThreadContext(),
-            features,
-            request,
-            version,
-            action,
-            requestId,
-            isHandshake,
-            compressRequest
-        );
-        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onRequestSent(node, requestId, action, request, options));
-        sendMessage(channel, message, listener);
-    }
-
-    /**
-     * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
-     * objects back to the caller.
-     *
-     * @see #sendErrorResponse(Version, Set, TcpChannel, long, String, Exception) for sending error responses
-     */
-    void sendResponse(
-        final Version nodeVersion,
-        final Set<String> features,
-        final TcpChannel channel,
-        final long requestId,
-        final String action,
-        final TransportResponse response,
-        final boolean compress,
-        final boolean isHandshake
-    ) throws IOException {
-        Version version = Version.min(this.version, nodeVersion);
-        OutboundMessage.Response message = new OutboundMessage.Response(
-            threadPool.getThreadContext(),
-            features,
-            response,
-            version,
-            requestId,
-            isHandshake,
-            compress
-        );
-        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response));
-        sendMessage(channel, message, listener);
-    }
-
-    /**
-     * Sends back an error response to the caller via the given channel
-     */
-    void sendErrorResponse(
-        final Version nodeVersion,
-        final Set<String> features,
-        final TcpChannel channel,
-        final long requestId,
-        final String action,
-        final Exception error
-    ) throws IOException {
-        Version version = Version.min(this.version, nodeVersion);
-        TransportAddress address = new TransportAddress(channel.getLocalAddress());
-        RemoteTransportException tx = new RemoteTransportException(nodeName, address, action, error);
-        OutboundMessage.Response message = new OutboundMessage.Response(
-            threadPool.getThreadContext(),
-            features,
-            tx,
-            version,
-            requestId,
-            false,
-            false
-        );
-        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, error));
-        sendMessage(channel, message, listener);
-    }
-
-    private void sendMessage(TcpChannel channel, OutboundMessage networkMessage, ActionListener<Void> listener) throws IOException {
-        MessageSerializer serializer = new MessageSerializer(networkMessage, bigArrays);
-        SendContext sendContext = new SendContext(channel, serializer, listener, serializer);
-        internalSend(channel, sendContext);
-    }
-
-    private void internalSend(TcpChannel channel, SendContext sendContext) throws IOException {
+    public void sendBytes(TcpChannel channel, SendContext sendContext) throws IOException {
         channel.getChannelStats().markAccessed(threadPool.relativeTimeInMillis());
         BytesReference reference = sendContext.get();
         // stash thread context so that channel event loop is not polluted by thread context
@@ -205,59 +88,30 @@ private void internalSend(TcpChannel channel, SendContext sendContext) throws IO
         }
     }
 
-    void setMessageListener(TransportMessageListener listener) {
-        if (messageListener == TransportMessageListener.NOOP_LISTENER) {
-            messageListener = listener;
-        } else {
-            throw new IllegalStateException("Cannot set message listener twice");
-        }
-    }
-
     /**
      * Internal message serializer
      *
      * @opensearch.internal
      */
-    private static class MessageSerializer implements CheckedSupplier<BytesReference, IOException>, Releasable {
-
-        private final OutboundMessage message;
-        private final BigArrays bigArrays;
-        private volatile ReleasableBytesStreamOutput bytesStreamOutput;
-
-        private MessageSerializer(OutboundMessage message, BigArrays bigArrays) {
-            this.message = message;
-            this.bigArrays = bigArrays;
-        }
-
-        @Override
-        public BytesReference get() throws IOException {
-            bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays);
-            return message.serialize(bytesStreamOutput);
-        }
-
-        @Override
-        public void close() {
-            IOUtils.closeWhileHandlingException(bytesStreamOutput);
-        }
-    }
-
-    private class SendContext extends NotifyOnceListener<Void> implements CheckedSupplier<BytesReference, IOException> {
-
+    public static class SendContext extends NotifyOnceListener<Void> implements CheckedSupplier<BytesReference, IOException> {
+        private final StatsTracker statsTracker;
         private final TcpChannel channel;
         private final CheckedSupplier<BytesReference, IOException> messageSupplier;
         private final ActionListener<Void> listener;
         private final Releasable optionalReleasable;
         private long messageSize = -1;
 
-        private SendContext(
+        SendContext(
+            StatsTracker statsTracker,
             TcpChannel channel,
             CheckedSupplier<BytesReference, IOException> messageSupplier,
             ActionListener<Void> listener
         ) {
-            this(channel, messageSupplier, listener, null);
+            this(statsTracker, channel, messageSupplier, listener, null);
         }
 
-        private SendContext(
+        public SendContext(
+            StatsTracker statsTracker,
             TcpChannel channel,
             CheckedSupplier<BytesReference, IOException> messageSupplier,
             ActionListener<Void> listener,
@@ -267,6 +121,7 @@ private SendContext(
             this.messageSupplier = messageSupplier;
             this.listener = listener;
             this.optionalReleasable = optionalReleasable;
+            this.statsTracker = statsTracker;
         }
 
         public BytesReference get() throws IOException {
diff --git a/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java b/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java
index 714d91d1e74c7..3c3fafebc34df 100644
--- a/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java
+++ b/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java
@@ -17,6 +17,14 @@
  */
 public interface ProtocolMessageHandler {
 
+    /**
+     * Handles the message received on the channel.
+     * @param channel the channel on which the message was received
+     * @param message the message received
+     * @param startTime the start time
+     * @param slowLogThresholdMs the threshold for slow logs
+     * @param messageListener the message listener
+     */
     public void messageReceived(
         TcpChannel channel,
         ProtocolInboundMessage message,
@@ -24,4 +32,10 @@ public void messageReceived(
         long slowLogThresholdMs,
         TransportMessageListener messageListener
     ) throws IOException;
+
+    /**
+     * Sets the message listener to be used by the handler.
+     * @param listener the message listener
+     */
+    public void setMessageListener(TransportMessageListener listener);
 }
diff --git a/server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java b/server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java
new file mode 100644
index 0000000000000..42c5462fddf80
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.core.transport.TransportResponse;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Protocol based outbound data handler.
+ * Different transport protocols can have different implementations of this class.
+ *
+ * @opensearch.internal
+ */
+public abstract class ProtocolOutboundHandler {
+
+    /**
+     * Sends the request to the given channel. This method should be used to send {@link TransportRequest}
+     * objects back to the caller.
+     */
+    public abstract void sendRequest(
+        final DiscoveryNode node,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportRequest request,
+        final TransportRequestOptions options,
+        final Version channelVersion,
+        final boolean compressRequest,
+        final boolean isHandshake
+    ) throws IOException, TransportException;
+
+    /**
+     * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
+     * objects back to the caller.
+     *
+     * @see #sendErrorResponse(Version, Set, TcpChannel, long, String, Exception) for sending error responses
+     */
+    public abstract void sendResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportResponse response,
+        final boolean compress,
+        final boolean isHandshake
+    ) throws IOException;
+
+    /**
+     * Sends back an error response to the caller via the given channel
+     */
+    public abstract void sendErrorResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final Exception error
+    ) throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java
index 8ba0178577232..ffa3168da0b3e 100644
--- a/server/src/main/java/org/opensearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java
@@ -71,6 +71,7 @@
 import org.opensearch.node.Node;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeOutboundHandler;
 
 import java.io.IOException;
 import java.io.StreamCorruptedException;
@@ -150,6 +151,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     private final TransportKeepAlive keepAlive;
     private final OutboundHandler outboundHandler;
     private final InboundHandler inboundHandler;
+    private final NativeOutboundHandler handshakerHandler;
     private final ResponseHandlers responseHandlers = new ResponseHandlers();
     private final RequestHandlers requestHandlers = new RequestHandlers();
 
@@ -188,11 +190,20 @@ public TcpTransport(
         }
         BigArrays bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService, CircuitBreaker.IN_FLIGHT_REQUESTS);
 
-        this.outboundHandler = new OutboundHandler(nodeName, version, features, statsTracker, threadPool, bigArrays);
+        this.outboundHandler = new OutboundHandler(statsTracker, threadPool);
+        this.handshakerHandler = new NativeOutboundHandler(
+            nodeName,
+            version,
+            features,
+            statsTracker,
+            threadPool,
+            bigArrays,
+            outboundHandler
+        );
         this.handshaker = new TransportHandshaker(
             version,
             threadPool,
-            (node, channel, requestId, v) -> outboundHandler.sendRequest(
+            (node, channel, requestId, v) -> handshakerHandler.sendRequest(
                 node,
                 channel,
                 requestId,
@@ -206,7 +217,12 @@ public TcpTransport(
         );
         this.keepAlive = new TransportKeepAlive(threadPool, this.outboundHandler::sendBytes);
         this.inboundHandler = new InboundHandler(
+            nodeName,
+            version,
+            features,
+            statsTracker,
             threadPool,
+            bigArrays,
             outboundHandler,
             namedWriteableRegistry,
             handshaker,
@@ -238,7 +254,7 @@ protected void doStart() {}
 
     @Override
     public synchronized void setMessageListener(TransportMessageListener listener) {
-        outboundHandler.setMessageListener(listener);
+        handshakerHandler.setMessageListener(listener);
         inboundHandler.setMessageListener(listener);
     }
 
@@ -319,7 +335,7 @@ public void sendRequest(long requestId, String action, TransportRequest request,
                 throw new NodeNotConnectedException(node, "connection already closed");
             }
             TcpChannel channel = channel(options.type());
-            outboundHandler.sendRequest(node, channel, requestId, action, request, options, getVersion(), compress, false);
+            handshakerHandler.sendRequest(node, channel, requestId, action, request, options, getVersion(), compress, false);
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
index 81de0af07ea7c..750fd50a4c44c 100644
--- a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
@@ -50,7 +50,7 @@
 public final class TcpTransportChannel extends BaseTcpTransportChannel {
 
     private final AtomicBoolean released = new AtomicBoolean();
-    private final OutboundHandler outboundHandler;
+    private final ProtocolOutboundHandler outboundHandler;
     private final String action;
     private final long requestId;
     private final Version version;
@@ -60,7 +60,7 @@ public final class TcpTransportChannel extends BaseTcpTransportChannel {
     private final Releasable breakerRelease;
 
     TcpTransportChannel(
-        OutboundHandler outboundHandler,
+        ProtocolOutboundHandler outboundHandler,
         TcpChannel channel,
         String action,
         long requestId,
diff --git a/server/src/main/java/org/opensearch/transport/TransportStatus.java b/server/src/main/java/org/opensearch/transport/TransportStatus.java
index dab572949e688..76377468535b9 100644
--- a/server/src/main/java/org/opensearch/transport/TransportStatus.java
+++ b/server/src/main/java/org/opensearch/transport/TransportStatus.java
@@ -76,11 +76,11 @@ public static byte setCompress(byte value) {
         return value;
     }
 
-    static boolean isHandshake(byte value) { // pkg private since it's only used internally
+    public static boolean isHandshake(byte value) {
         return (value & STATUS_HANDSHAKE) != 0;
     }
 
-    static byte setHandshake(byte value) { // pkg private since it's only used internally
+    public static byte setHandshake(byte value) {
         value |= STATUS_HANDSHAKE;
         return value;
     }
diff --git a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStream.java
similarity index 98%
rename from server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStream.java
index 57707d3b44477..92b682370bcd5 100644
--- a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStream.java
@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.common.io.Streams;
 import org.opensearch.common.util.io.IOUtils;
diff --git a/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java
new file mode 100644
index 0000000000000..66ed0d8e3eb2b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java
@@ -0,0 +1,224 @@
+/*
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+/*
+* Licensed to Elasticsearch under one or more contributor
+* license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright
+* ownership. Elasticsearch licenses this file to you under
+* the Apache License, Version 2.0 (the "License"); you may
+* not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*/
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.opensearch.common.lease.Releasable;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.transport.TransportResponse;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.OutboundHandler;
+import org.opensearch.transport.ProtocolOutboundHandler;
+import org.opensearch.transport.RemoteTransportException;
+import org.opensearch.transport.StatsTracker;
+import org.opensearch.transport.TcpChannel;
+import org.opensearch.transport.TransportException;
+import org.opensearch.transport.TransportMessageListener;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportRequestOptions;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Outbound data handler
+ *
+ * @opensearch.internal
+ */
+public final class NativeOutboundHandler extends ProtocolOutboundHandler {
+    private final String nodeName;
+    private final Version version;
+    private final String[] features;
+    private final StatsTracker statsTracker;
+    private final ThreadPool threadPool;
+    private final BigArrays bigArrays;
+    private volatile TransportMessageListener messageListener = TransportMessageListener.NOOP_LISTENER;
+    private final OutboundHandler handler;
+
+    public NativeOutboundHandler(
+        String nodeName,
+        Version version,
+        String[] features,
+        StatsTracker statsTracker,
+        ThreadPool threadPool,
+        BigArrays bigArrays,
+        OutboundHandler handler
+    ) {
+        this.nodeName = nodeName;
+        this.version = version;
+        this.features = features;
+        this.statsTracker = statsTracker;
+        this.threadPool = threadPool;
+        this.bigArrays = bigArrays;
+        this.handler = handler;
+    }
+
+    /**
+     * Sends the request to the given channel. This method should be used to send {@link TransportRequest}
+     * objects back to the caller.
+     */
+    @Override
+    public void sendRequest(
+        final DiscoveryNode node,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportRequest request,
+        final TransportRequestOptions options,
+        final Version channelVersion,
+        final boolean compressRequest,
+        final boolean isHandshake
+    ) throws IOException, TransportException {
+        Version version = Version.min(this.version, channelVersion);
+        NativeOutboundMessage.Request message = new NativeOutboundMessage.Request(
+            threadPool.getThreadContext(),
+            features,
+            request,
+            version,
+            action,
+            requestId,
+            isHandshake,
+            compressRequest
+        );
+        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onRequestSent(node, requestId, action, request, options));
+        sendMessage(channel, message, listener);
+    }
+
+    /**
+     * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
+     * objects back to the caller.
+     *
+     * @see #sendErrorResponse(Version, Set, TcpChannel, long, String, Exception) for sending error responses
+     */
+    @Override
+    public void sendResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportResponse response,
+        final boolean compress,
+        final boolean isHandshake
+    ) throws IOException {
+        Version version = Version.min(this.version, nodeVersion);
+        NativeOutboundMessage.Response message = new NativeOutboundMessage.Response(
+            threadPool.getThreadContext(),
+            features,
+            response,
+            version,
+            requestId,
+            isHandshake,
+            compress
+        );
+        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response));
+        sendMessage(channel, message, listener);
+    }
+
+    /**
+     * Sends back an error response to the caller via the given channel
+     */
+    @Override
+    public void sendErrorResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final Exception error
+    ) throws IOException {
+        Version version = Version.min(this.version, nodeVersion);
+        TransportAddress address = new TransportAddress(channel.getLocalAddress());
+        RemoteTransportException tx = new RemoteTransportException(nodeName, address, action, error);
+        NativeOutboundMessage.Response message = new NativeOutboundMessage.Response(
+            threadPool.getThreadContext(),
+            features,
+            tx,
+            version,
+            requestId,
+            false,
+            false
+        );
+        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, error));
+        sendMessage(channel, message, listener);
+    }
+
+    private void sendMessage(TcpChannel channel, NativeOutboundMessage networkMessage, ActionListener<Void> listener) throws IOException {
+        MessageSerializer serializer = new MessageSerializer(networkMessage, bigArrays);
+        OutboundHandler.SendContext sendContext = new OutboundHandler.SendContext(statsTracker, channel, serializer, listener, serializer);
+        handler.sendBytes(channel, sendContext);
+    }
+
+    public void setMessageListener(TransportMessageListener listener) {
+        if (messageListener == TransportMessageListener.NOOP_LISTENER) {
+            messageListener = listener;
+        } else {
+            throw new IllegalStateException("Cannot set message listener twice");
+        }
+    }
+
+    /**
+     * Internal message serializer
+     *
+     * @opensearch.internal
+     */
+    private static class MessageSerializer implements CheckedSupplier<BytesReference, IOException>, Releasable {
+
+        private final NativeOutboundMessage message;
+        private final BigArrays bigArrays;
+        private volatile ReleasableBytesStreamOutput bytesStreamOutput;
+
+        private MessageSerializer(NativeOutboundMessage message, BigArrays bigArrays) {
+            this.message = message;
+            this.bigArrays = bigArrays;
+        }
+
+        @Override
+        public BytesReference get() throws IOException {
+            bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays);
+            return message.serialize(bytesStreamOutput);
+        }
+
+        @Override
+        public void close() {
+            IOUtils.closeWhileHandlingException(bytesStreamOutput);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/transport/OutboundMessage.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessage.java
similarity index 91%
rename from server/src/main/java/org/opensearch/transport/OutboundMessage.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessage.java
index 3dafc6fb5eea9..d7590fb9e03ab 100644
--- a/server/src/main/java/org/opensearch/transport/OutboundMessage.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessage.java
@@ -29,7 +29,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.Version;
 import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -39,6 +39,10 @@
 import org.opensearch.core.common.bytes.CompositeBytesReference;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.BytesTransportRequest;
+import org.opensearch.transport.RemoteTransportException;
+import org.opensearch.transport.TcpHeader;
+import org.opensearch.transport.TransportStatus;
 
 import java.io.IOException;
 import java.util.Set;
@@ -48,11 +52,11 @@
  *
  * @opensearch.internal
  */
-abstract class OutboundMessage extends NetworkMessage {
+abstract class NativeOutboundMessage extends NetworkMessage {
 
     private final Writeable message;
 
-    OutboundMessage(ThreadContext threadContext, Version version, byte status, long requestId, Writeable message) {
+    NativeOutboundMessage(ThreadContext threadContext, Version version, byte status, long requestId, Writeable message) {
         super(threadContext, version, status, requestId);
         this.message = message;
     }
@@ -93,7 +97,7 @@ protected BytesReference writeMessage(CompressibleBytesOutputStream stream) thro
         if (message instanceof BytesTransportRequest) {
             BytesTransportRequest bRequest = (BytesTransportRequest) message;
             bRequest.writeThin(stream);
-            zeroCopyBuffer = bRequest.bytes;
+            zeroCopyBuffer = bRequest.bytes();
         } else if (message instanceof RemoteTransportException) {
             stream.writeException((RemoteTransportException) message);
             zeroCopyBuffer = BytesArray.EMPTY;
@@ -119,7 +123,7 @@ protected BytesReference writeMessage(CompressibleBytesOutputStream stream) thro
      *
      * @opensearch.internal
      */
-    static class Request extends OutboundMessage {
+    static class Request extends NativeOutboundMessage {
 
         private final String[] features;
         private final String action;
@@ -149,7 +153,7 @@ protected void writeVariableHeader(StreamOutput stream) throws IOException {
         private static byte setStatus(boolean compress, boolean isHandshake, Writeable message) {
             byte status = 0;
             status = TransportStatus.setRequest(status);
-            if (compress && OutboundMessage.canCompress(message)) {
+            if (compress && NativeOutboundMessage.canCompress(message)) {
                 status = TransportStatus.setCompress(status);
             }
             if (isHandshake) {
@@ -165,7 +169,7 @@ private static byte setStatus(boolean compress, boolean isHandshake, Writeable m
      *
      * @opensearch.internal
      */
-    static class Response extends OutboundMessage {
+    static class Response extends NativeOutboundMessage {
 
         private final Set<String> features;
 
diff --git a/server/src/main/java/org/opensearch/transport/NetworkMessage.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NetworkMessage.java
similarity index 96%
rename from server/src/main/java/org/opensearch/transport/NetworkMessage.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/NetworkMessage.java
index f02d664b65929..c197539d2e009 100644
--- a/server/src/main/java/org/opensearch/transport/NetworkMessage.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NetworkMessage.java
@@ -29,11 +29,12 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.Version;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.TransportStatus;
 
 /**
  * Represents a transport message sent over the network. Subclasses implement serialization and
diff --git a/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java b/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java
index 4d671443f396e..47ae8f9783ea3 100644
--- a/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java
@@ -39,19 +39,29 @@
 import org.opensearch.common.util.PageCacheRecycler;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.transport.TransportMessage;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.hasItems;
 
-public class InboundDecoderTests extends OpenSearchTestCase {
+public abstract class InboundDecoderTests extends OpenSearchTestCase {
 
-    private ThreadContext threadContext;
+    protected ThreadContext threadContext;
+
+    protected abstract BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        Writeable transportMessage
+    ) throws IOException;
 
     @Override
     public void setUp() throws Exception {
@@ -65,36 +75,16 @@ public void testDecode() throws IOException {
         long requestId = randomNonNegativeLong();
         final String headerKey = randomAlphaOfLength(10);
         final String headerValue = randomAlphaOfLength(20);
+        TransportMessage transportMessage;
         if (isRequest) {
             threadContext.putHeader(headerKey, headerValue);
+            transportMessage = new TestRequest(randomAlphaOfLength(100));
         } else {
             threadContext.addResponseHeader(headerKey, headerValue);
-        }
-        OutboundMessage message;
-        if (isRequest) {
-            message = new OutboundMessage.Request(
-                threadContext,
-                new String[0],
-                new TestRequest(randomAlphaOfLength(100)),
-                Version.CURRENT,
-                action,
-                requestId,
-                false,
-                false
-            );
-        } else {
-            message = new OutboundMessage.Response(
-                threadContext,
-                Collections.emptySet(),
-                new TestResponse(randomAlphaOfLength(100)),
-                Version.CURRENT,
-                requestId,
-                false,
-                false
-            );
+            transportMessage = new TestResponse(randomAlphaOfLength(100));
         }
 
-        final BytesReference totalBytes = message.serialize(new BytesStreamOutput());
+        final BytesReference totalBytes = serialize(isRequest, Version.CURRENT, false, false, action, requestId, transportMessage);
         int totalHeaderSize = TcpHeader.headerSize(Version.CURRENT) + totalBytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
         final BytesReference messageBytes = totalBytes.slice(totalHeaderSize, totalBytes.length() - totalHeaderSize);
 
@@ -142,18 +132,16 @@ public void testDecodeHandshakeCompatibility() throws IOException {
         final String headerValue = randomAlphaOfLength(20);
         threadContext.putHeader(headerKey, headerValue);
         Version handshakeCompatVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
-        OutboundMessage message = new OutboundMessage.Request(
-            threadContext,
-            new String[0],
-            new TestRequest(randomAlphaOfLength(100)),
+
+        final BytesReference bytes = serialize(
+            true,
             handshakeCompatVersion,
+            true,
+            false,
             action,
             requestId,
-            true,
-            false
+            new TestRequest(randomAlphaOfLength(100))
         );
-
-        final BytesReference bytes = message.serialize(new BytesStreamOutput());
         int totalHeaderSize = TcpHeader.headerSize(handshakeCompatVersion) + bytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
 
         InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
@@ -183,34 +171,14 @@ public void testCompressedDecode() throws IOException {
         } else {
             threadContext.addResponseHeader(headerKey, headerValue);
         }
-        OutboundMessage message;
         TransportMessage transportMessage;
         if (isRequest) {
             transportMessage = new TestRequest(randomAlphaOfLength(100));
-            message = new OutboundMessage.Request(
-                threadContext,
-                new String[0],
-                transportMessage,
-                Version.CURRENT,
-                action,
-                requestId,
-                false,
-                true
-            );
         } else {
             transportMessage = new TestResponse(randomAlphaOfLength(100));
-            message = new OutboundMessage.Response(
-                threadContext,
-                Collections.emptySet(),
-                transportMessage,
-                Version.CURRENT,
-                requestId,
-                false,
-                true
-            );
         }
 
-        final BytesReference totalBytes = message.serialize(new BytesStreamOutput());
+        final BytesReference totalBytes = serialize(isRequest, Version.CURRENT, false, true, action, requestId, transportMessage);
         final BytesStreamOutput out = new BytesStreamOutput();
         transportMessage.writeTo(out);
         final BytesReference uncompressedBytes = out.bytes();
@@ -260,18 +228,16 @@ public void testCompressedDecodeHandshakeCompatibility() throws IOException {
         final String headerValue = randomAlphaOfLength(20);
         threadContext.putHeader(headerKey, headerValue);
         Version handshakeCompatVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
-        OutboundMessage message = new OutboundMessage.Request(
-            threadContext,
-            new String[0],
-            new TestRequest(randomAlphaOfLength(100)),
+
+        final BytesReference bytes = serialize(
+            true,
             handshakeCompatVersion,
+            true,
+            true,
             action,
             requestId,
-            true,
-            true
+            new TestRequest(randomAlphaOfLength(100))
         );
-
-        final BytesReference bytes = message.serialize(new BytesStreamOutput());
         int totalHeaderSize = TcpHeader.headerSize(handshakeCompatVersion) + bytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
 
         InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
@@ -294,19 +260,17 @@ public void testVersionIncompatibilityDecodeException() throws IOException {
         String action = "test-request";
         long requestId = randomNonNegativeLong();
         Version incompatibleVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
-        OutboundMessage message = new OutboundMessage.Request(
-            threadContext,
-            new String[0],
-            new TestRequest(randomAlphaOfLength(100)),
+
+        final BytesReference bytes = serialize(
+            true,
             incompatibleVersion,
+            false,
+            true,
             action,
             requestId,
-            false,
-            true
+            new TestRequest(randomAlphaOfLength(100))
         );
 
-        final BytesReference bytes = message.serialize(new BytesStreamOutput());
-
         InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
         final ArrayList<Object> fragments = new ArrayList<>();
         final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(bytes);
diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java
index 2dde27d62e759..2553e7740990b 100644
--- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java
@@ -39,16 +39,17 @@
 import org.opensearch.Version;
 import org.opensearch.common.bytes.ReleasableBytesReference;
 import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.InputStreamStreamInput;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.tasks.TaskManager;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.MockLogAppender;
@@ -75,7 +76,17 @@
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.instanceOf;
 
-public class InboundHandlerTests extends OpenSearchTestCase {
+public abstract class InboundHandlerTests extends OpenSearchTestCase {
+
+    public abstract BytesReference serializeOutboundRequest(
+        ThreadContext threadContext,
+        Writeable message,
+        Version version,
+        String action,
+        long requestId,
+        boolean compress,
+        boolean handshake
+    ) throws IOException;
 
     private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
     private final Version version = Version.CURRENT;
@@ -101,19 +112,17 @@ public void sendMessage(BytesReference reference, ActionListener<Void> listener)
         };
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         TransportHandshaker handshaker = new TransportHandshaker(version, threadPool, (n, c, r, v) -> {});
-        outboundHandler = new OutboundHandler(
-            "node",
-            version,
-            new String[0],
-            new StatsTracker(),
-            threadPool,
-            BigArrays.NON_RECYCLING_INSTANCE
-        );
+        outboundHandler = new OutboundHandler(new StatsTracker(), threadPool);
         TransportKeepAlive keepAlive = new TransportKeepAlive(threadPool, outboundHandler::sendBytes);
         requestHandlers = new Transport.RequestHandlers();
         responseHandlers = new Transport.ResponseHandlers();
         handler = new InboundHandler(
+            "node",
+            version,
+            new String[0],
+            new StatsTracker(),
             threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
             outboundHandler,
             namedWriteableRegistry,
             handshaker,
@@ -195,9 +204,9 @@ public TestResponse read(StreamInput in) throws IOException {
         );
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -205,8 +214,6 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
         NativeInboundMessage requestMessage = new NativeInboundMessage(
@@ -393,18 +400,8 @@ public TestResponse read(StreamInput in) throws IOException {
 
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
-            threadPool.getThreadContext(),
-            new String[0],
-            new TestRequest(requestValue),
-            version,
-            action,
-            requestId,
-            false,
-            false
-        );
 
-        outboundHandler.setMessageListener(new TransportMessageListener() {
+        handler.setMessageListener(new TransportMessageListener() {
             @Override
             public void onResponseSent(long requestId, String action, Exception error) {
                 exceptionCaptor.set(error);
@@ -412,7 +409,15 @@ public void onResponseSent(long requestId, String action, Exception error) {
         });
 
         // Create the request payload with 1 byte overflow
-        final BytesRef bytes = request.serialize(new BytesStreamOutput()).toBytesRef();
+        final BytesRef bytes = serializeOutboundRequest(
+            threadPool.getThreadContext(),
+            new TestRequest(requestValue),
+            version,
+            action,
+            requestId,
+            false,
+            false
+        ).toBytesRef();
         final ByteBuffer buffer = ByteBuffer.allocate(bytes.length + 1);
         buffer.put(bytes.bytes, 0, bytes.length);
         buffer.put((byte) 1);
@@ -469,9 +474,16 @@ public TestResponse read(StreamInput in) throws IOException {
 
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        handler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onResponseSent(long requestId, String action, Exception error) {
+                exceptionCaptor.set(error);
+            }
+        });
+
+        final BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -479,15 +491,6 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        outboundHandler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onResponseSent(long requestId, String action, Exception error) {
-                exceptionCaptor.set(error);
-            }
-        });
-
-        final BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         // Create the request payload by intentionally stripping 1 byte away
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize - 1);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
@@ -547,9 +550,9 @@ public TestResponse read(StreamInput in) throws IOException {
         );
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -557,8 +560,6 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
         NativeInboundMessage requestMessage = new NativeInboundMessage(
@@ -643,9 +644,9 @@ public TestResponse read(StreamInput in) throws IOException {
         );
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -653,8 +654,6 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
         NativeInboundMessage requestMessage = new NativeInboundMessage(
diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java
index d54f7e6fd2c2b..5a89bf1e0ead3 100644
--- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java
@@ -53,7 +53,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -64,12 +63,22 @@
 
 import static org.hamcrest.Matchers.instanceOf;
 
-public class InboundPipelineTests extends OpenSearchTestCase {
+public abstract class InboundPipelineTests extends OpenSearchTestCase {
 
     private static final int BYTE_THRESHOLD = 128 * 1024;
-    private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-
-    public void testPipelineHandling() throws IOException {
+    public final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+    protected abstract BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        String value
+    ) throws IOException;
+
+    public void testPipelineHandlingForNativeProtocol() throws IOException {
         final List<Tuple<MessageData, Exception>> expected = new ArrayList<>();
         final List<Tuple<MessageData, Exception>> actual = new ArrayList<>();
         final List<ReleasableBytesReference> toRelease = new ArrayList<>();
@@ -86,10 +95,10 @@ public void testPipelineHandling() throws IOException {
                     actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), null);
                 } else if (isRequest) {
                     final TestRequest request = new TestRequest(message.openOrGetStreamInput());
-                    actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), request.value);
+                    actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), request.getValue());
                 } else {
                     final TestResponse response = new TestResponse(message.openOrGetStreamInput());
-                    actualData = new MessageData(version, requestId, isRequest, isCompressed, null, response.value);
+                    actualData = new MessageData(version, requestId, isRequest, isCompressed, null, response.getValue());
                 }
                 actual.add(new Tuple<>(actualData, message.getException()));
             } catch (IOException e) {
@@ -128,49 +137,23 @@ public void testPipelineHandling() throws IOException {
                     final MessageData messageData;
                     Exception expectedExceptionClass = null;
 
-                    OutboundMessage message;
+                    // NativeOutboundMessage message;
+                    final BytesReference reference;
                     if (isRequest) {
                         if (rarely()) {
                             messageData = new MessageData(version, requestId, true, isCompressed, breakThisAction, null);
-                            message = new OutboundMessage.Request(
-                                threadContext,
-                                new String[0],
-                                new TestRequest(value),
-                                version,
-                                breakThisAction,
-                                requestId,
-                                false,
-                                isCompressed
-                            );
+                            reference = serialize(true, version, false, isCompressed, breakThisAction, requestId, value);
                             expectedExceptionClass = new CircuitBreakingException("", CircuitBreaker.Durability.PERMANENT);
                         } else {
                             messageData = new MessageData(version, requestId, true, isCompressed, actionName, value);
-                            message = new OutboundMessage.Request(
-                                threadContext,
-                                new String[0],
-                                new TestRequest(value),
-                                version,
-                                actionName,
-                                requestId,
-                                false,
-                                isCompressed
-                            );
+                            reference = serialize(true, version, false, isCompressed, actionName, requestId, value);
                         }
                     } else {
                         messageData = new MessageData(version, requestId, false, isCompressed, null, value);
-                        message = new OutboundMessage.Response(
-                            threadContext,
-                            Collections.emptySet(),
-                            new TestResponse(value),
-                            version,
-                            requestId,
-                            false,
-                            isCompressed
-                        );
+                        reference = serialize(false, version, false, isCompressed, actionName, requestId, value);
                     }
 
                     expected.add(new Tuple<>(messageData, expectedExceptionClass));
-                    final BytesReference reference = message.serialize(new BytesStreamOutput());
                     Streams.copy(reference.streamInput(), streamOutput);
                 }
 
@@ -231,31 +214,7 @@ public void testDecodeExceptionIsPropagated() throws IOException {
             final boolean isRequest = randomBoolean();
             final long requestId = randomNonNegativeLong();
 
-            OutboundMessage message;
-            if (isRequest) {
-                message = new OutboundMessage.Request(
-                    threadContext,
-                    new String[0],
-                    new TestRequest(value),
-                    invalidVersion,
-                    actionName,
-                    requestId,
-                    false,
-                    false
-                );
-            } else {
-                message = new OutboundMessage.Response(
-                    threadContext,
-                    Collections.emptySet(),
-                    new TestResponse(value),
-                    invalidVersion,
-                    requestId,
-                    false,
-                    false
-                );
-            }
-
-            final BytesReference reference = message.serialize(streamOutput);
+            final BytesReference reference = serialize(isRequest, invalidVersion, false, false, actionName, requestId, value);
             try (ReleasableBytesReference releasable = ReleasableBytesReference.wrap(reference)) {
                 expectThrows(IllegalStateException.class, () -> pipeline.handleBytes(new FakeTcpChannel(), releasable));
             }
@@ -285,31 +244,7 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
             final boolean isRequest = randomBoolean();
             final long requestId = randomNonNegativeLong();
 
-            OutboundMessage message;
-            if (isRequest) {
-                message = new OutboundMessage.Request(
-                    threadContext,
-                    new String[0],
-                    new TestRequest(value),
-                    version,
-                    actionName,
-                    requestId,
-                    false,
-                    false
-                );
-            } else {
-                message = new OutboundMessage.Response(
-                    threadContext,
-                    Collections.emptySet(),
-                    new TestResponse(value),
-                    version,
-                    requestId,
-                    false,
-                    false
-                );
-            }
-
-            final BytesReference reference = message.serialize(streamOutput);
+            final BytesReference reference = serialize(isRequest, version, false, false, actionName, requestId, value);
             final int fixedHeaderSize = TcpHeader.headerSize(Version.CURRENT);
             final int variableHeaderSize = reference.getInt(fixedHeaderSize - 4);
             final int totalHeaderSize = fixedHeaderSize + variableHeaderSize;
diff --git a/server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java
new file mode 100644
index 0000000000000..01f19bea7a37f
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java
@@ -0,0 +1,302 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.bytes.ReleasableBytesReference;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.PageCacheRecycler;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.common.util.io.Streams;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.transport.TransportResponse;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
+import org.opensearch.transport.nativeprotocol.NativeOutboundHandler;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.instanceOf;
+
+public class NativeOutboundHandlerTests extends OpenSearchTestCase {
+
+    private final String feature1 = "feature1";
+    private final String feature2 = "feature2";
+    private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
+    private final TransportRequestOptions options = TransportRequestOptions.EMPTY;
+    private final AtomicReference<Tuple<Header, BytesReference>> message = new AtomicReference<>();
+    private InboundPipeline pipeline;
+    private OutboundHandler handler;
+    private NativeOutboundHandler nativeOutboundHandler;
+    private FakeTcpChannel channel;
+    private DiscoveryNode node;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        channel = new FakeTcpChannel(randomBoolean(), buildNewFakeTransportAddress().address(), buildNewFakeTransportAddress().address());
+        TransportAddress transportAddress = buildNewFakeTransportAddress();
+        node = new DiscoveryNode("", transportAddress, Version.CURRENT);
+        String[] features = { feature1, feature2 };
+        StatsTracker statsTracker = new StatsTracker();
+        handler = new OutboundHandler(statsTracker, threadPool);
+        nativeOutboundHandler = new NativeOutboundHandler(
+            "node",
+            Version.CURRENT,
+            features,
+            statsTracker,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            handler
+        );
+
+        final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
+        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
+        final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
+        final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
+        pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> {
+            try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
+                NativeInboundMessage m1 = (NativeInboundMessage) m;
+                Streams.copy(m1.openOrGetStreamInput(), streamOutput);
+                message.set(new Tuple<>(m1.getHeader(), streamOutput.bytes()));
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        });
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+        super.tearDown();
+    }
+
+    public void testSendRequest() throws IOException {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        String action = "handshake";
+        long requestId = randomLongBetween(0, 300);
+        boolean isHandshake = randomBoolean();
+        boolean compress = randomBoolean();
+        String value = "message";
+        threadContext.putHeader("header", "header_value");
+        TestRequest request = new TestRequest(value);
+
+        AtomicReference<DiscoveryNode> nodeRef = new AtomicReference<>();
+        AtomicLong requestIdRef = new AtomicLong();
+        AtomicReference<String> actionRef = new AtomicReference<>();
+        AtomicReference<TransportRequest> requestRef = new AtomicReference<>();
+        nativeOutboundHandler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onRequestSent(
+                DiscoveryNode node,
+                long requestId,
+                String action,
+                TransportRequest request,
+                TransportRequestOptions options
+            ) {
+                nodeRef.set(node);
+                requestIdRef.set(requestId);
+                actionRef.set(action);
+                requestRef.set(request);
+            }
+        });
+        nativeOutboundHandler.sendRequest(node, channel, requestId, action, request, options, version, compress, isHandshake);
+
+        BytesReference reference = channel.getMessageCaptor().get();
+        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
+        if (randomBoolean()) {
+            sendListener.onResponse(null);
+        } else {
+            sendListener.onFailure(new IOException("failed"));
+        }
+        assertEquals(node, nodeRef.get());
+        assertEquals(requestId, requestIdRef.get());
+        assertEquals(action, actionRef.get());
+        assertEquals(request, requestRef.get());
+
+        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
+        final Tuple<Header, BytesReference> tuple = message.get();
+        final Header header = tuple.v1();
+        final TestRequest message = new TestRequest(tuple.v2().streamInput());
+        assertEquals(version, header.getVersion());
+        assertEquals(requestId, header.getRequestId());
+        assertTrue(header.isRequest());
+        assertFalse(header.isResponse());
+        if (isHandshake) {
+            assertTrue(header.isHandshake());
+        } else {
+            assertFalse(header.isHandshake());
+        }
+        if (compress) {
+            assertTrue(header.isCompressed());
+        } else {
+            assertFalse(header.isCompressed());
+        }
+
+        assertEquals(value, message.getValue());
+        assertEquals("header_value", header.getHeaders().v1().get("header"));
+    }
+
+    public void testSendResponse() throws IOException {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        String action = "handshake";
+        long requestId = randomLongBetween(0, 300);
+        boolean isHandshake = randomBoolean();
+        boolean compress = randomBoolean();
+        String value = "message";
+        threadContext.putHeader("header", "header_value");
+        TestResponse response = new TestResponse(value);
+
+        AtomicLong requestIdRef = new AtomicLong();
+        AtomicReference<String> actionRef = new AtomicReference<>();
+        AtomicReference<TransportResponse> responseRef = new AtomicReference<>();
+        nativeOutboundHandler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onResponseSent(long requestId, String action, TransportResponse response) {
+                requestIdRef.set(requestId);
+                actionRef.set(action);
+                responseRef.set(response);
+            }
+        });
+        nativeOutboundHandler.sendResponse(version, Collections.emptySet(), channel, requestId, action, response, compress, isHandshake);
+
+        BytesReference reference = channel.getMessageCaptor().get();
+        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
+        if (randomBoolean()) {
+            sendListener.onResponse(null);
+        } else {
+            sendListener.onFailure(new IOException("failed"));
+        }
+        assertEquals(requestId, requestIdRef.get());
+        assertEquals(action, actionRef.get());
+        assertEquals(response, responseRef.get());
+
+        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
+        final Tuple<Header, BytesReference> tuple = message.get();
+        final Header header = tuple.v1();
+        final TestResponse message = new TestResponse(tuple.v2().streamInput());
+        assertEquals(version, header.getVersion());
+        assertEquals(requestId, header.getRequestId());
+        assertFalse(header.isRequest());
+        assertTrue(header.isResponse());
+        if (isHandshake) {
+            assertTrue(header.isHandshake());
+        } else {
+            assertFalse(header.isHandshake());
+        }
+        if (compress) {
+            assertTrue(header.isCompressed());
+        } else {
+            assertFalse(header.isCompressed());
+        }
+
+        assertFalse(header.isError());
+
+        assertEquals(value, message.getValue());
+        assertEquals("header_value", header.getHeaders().v1().get("header"));
+    }
+
+    public void testErrorResponse() throws IOException {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        String action = "handshake";
+        long requestId = randomLongBetween(0, 300);
+        threadContext.putHeader("header", "header_value");
+        OpenSearchException error = new OpenSearchException("boom");
+
+        AtomicLong requestIdRef = new AtomicLong();
+        AtomicReference<String> actionRef = new AtomicReference<>();
+        AtomicReference<Exception> responseRef = new AtomicReference<>();
+        nativeOutboundHandler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onResponseSent(long requestId, String action, Exception error) {
+                requestIdRef.set(requestId);
+                actionRef.set(action);
+                responseRef.set(error);
+            }
+        });
+        nativeOutboundHandler.sendErrorResponse(version, Collections.emptySet(), channel, requestId, action, error);
+
+        BytesReference reference = channel.getMessageCaptor().get();
+        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
+        if (randomBoolean()) {
+            sendListener.onResponse(null);
+        } else {
+            sendListener.onFailure(new IOException("failed"));
+        }
+        assertEquals(requestId, requestIdRef.get());
+        assertEquals(action, actionRef.get());
+        assertEquals(error, responseRef.get());
+
+        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
+        final Tuple<Header, BytesReference> tuple = message.get();
+        final Header header = tuple.v1();
+        assertEquals(version, header.getVersion());
+        assertEquals(requestId, header.getRequestId());
+        assertFalse(header.isRequest());
+        assertTrue(header.isResponse());
+        assertFalse(header.isCompressed());
+        assertFalse(header.isHandshake());
+        assertTrue(header.isError());
+
+        RemoteTransportException remoteException = tuple.v2().streamInput().readException();
+        assertThat(remoteException.getCause(), instanceOf(OpenSearchException.class));
+        assertEquals(remoteException.getCause().getMessage(), "boom");
+        assertEquals(action, remoteException.action());
+        assertEquals(channel.getLocalAddress(), remoteException.address().address());
+
+        assertEquals("header_value", header.getHeaders().v1().get("header"));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java
index ad7d4401af13c..7e7c60e2d3d29 100644
--- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java
+++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java
@@ -32,79 +32,33 @@
 
 package org.opensearch.transport;
 
-import org.opensearch.OpenSearchException;
-import org.opensearch.Version;
-import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.bytes.ReleasableBytesReference;
-import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.BigArrays;
-import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.common.util.io.Streams;
 import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.common.breaker.CircuitBreaker;
-import org.opensearch.core.common.breaker.NoopCircuitBreaker;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.transport.TransportAddress;
-import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
-import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.LongSupplier;
-import java.util.function.Predicate;
-import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.instanceOf;
 
 public class OutboundHandlerTests extends OpenSearchTestCase {
 
-    private final String feature1 = "feature1";
-    private final String feature2 = "feature2";
     private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
-    private final TransportRequestOptions options = TransportRequestOptions.EMPTY;
-    private final AtomicReference<Tuple<Header, BytesReference>> message = new AtomicReference<>();
-    private InboundPipeline pipeline;
     private OutboundHandler handler;
     private FakeTcpChannel channel;
-    private DiscoveryNode node;
 
     @Before
     public void setUp() throws Exception {
         super.setUp();
         channel = new FakeTcpChannel(randomBoolean(), buildNewFakeTransportAddress().address(), buildNewFakeTransportAddress().address());
-        TransportAddress transportAddress = buildNewFakeTransportAddress();
-        node = new DiscoveryNode("", transportAddress, Version.CURRENT);
-        String[] features = { feature1, feature2 };
         StatsTracker statsTracker = new StatsTracker();
-        handler = new OutboundHandler("node", Version.CURRENT, features, statsTracker, threadPool, BigArrays.NON_RECYCLING_INSTANCE);
-
-        final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
-        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
-        final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
-        final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
-        pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> {
-            try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
-                NativeInboundMessage m1 = (NativeInboundMessage) m;
-                Streams.copy(m1.openOrGetStreamInput(), streamOutput);
-                message.set(new Tuple<>(m1.getHeader(), streamOutput.bytes()));
-            } catch (IOException e) {
-                throw new AssertionError(e);
-            }
-        });
+        handler = new OutboundHandler(statsTracker, threadPool);
     }
 
     @After
@@ -137,182 +91,4 @@ public void testSendRawBytes() {
         assertEquals(bytesArray, reference);
     }
 
-    public void testSendRequest() throws IOException {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
-        String action = "handshake";
-        long requestId = randomLongBetween(0, 300);
-        boolean isHandshake = randomBoolean();
-        boolean compress = randomBoolean();
-        String value = "message";
-        threadContext.putHeader("header", "header_value");
-        TestRequest request = new TestRequest(value);
-
-        AtomicReference<DiscoveryNode> nodeRef = new AtomicReference<>();
-        AtomicLong requestIdRef = new AtomicLong();
-        AtomicReference<String> actionRef = new AtomicReference<>();
-        AtomicReference<TransportRequest> requestRef = new AtomicReference<>();
-        handler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onRequestSent(
-                DiscoveryNode node,
-                long requestId,
-                String action,
-                TransportRequest request,
-                TransportRequestOptions options
-            ) {
-                nodeRef.set(node);
-                requestIdRef.set(requestId);
-                actionRef.set(action);
-                requestRef.set(request);
-            }
-        });
-        handler.sendRequest(node, channel, requestId, action, request, options, version, compress, isHandshake);
-
-        BytesReference reference = channel.getMessageCaptor().get();
-        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
-        if (randomBoolean()) {
-            sendListener.onResponse(null);
-        } else {
-            sendListener.onFailure(new IOException("failed"));
-        }
-        assertEquals(node, nodeRef.get());
-        assertEquals(requestId, requestIdRef.get());
-        assertEquals(action, actionRef.get());
-        assertEquals(request, requestRef.get());
-
-        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
-        final Tuple<Header, BytesReference> tuple = message.get();
-        final Header header = tuple.v1();
-        final TestRequest message = new TestRequest(tuple.v2().streamInput());
-        assertEquals(version, header.getVersion());
-        assertEquals(requestId, header.getRequestId());
-        assertTrue(header.isRequest());
-        assertFalse(header.isResponse());
-        if (isHandshake) {
-            assertTrue(header.isHandshake());
-        } else {
-            assertFalse(header.isHandshake());
-        }
-        if (compress) {
-            assertTrue(header.isCompressed());
-        } else {
-            assertFalse(header.isCompressed());
-        }
-
-        assertEquals(value, message.value);
-        assertEquals("header_value", header.getHeaders().v1().get("header"));
-    }
-
-    public void testSendResponse() throws IOException {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
-        String action = "handshake";
-        long requestId = randomLongBetween(0, 300);
-        boolean isHandshake = randomBoolean();
-        boolean compress = randomBoolean();
-        String value = "message";
-        threadContext.putHeader("header", "header_value");
-        TestResponse response = new TestResponse(value);
-
-        AtomicLong requestIdRef = new AtomicLong();
-        AtomicReference<String> actionRef = new AtomicReference<>();
-        AtomicReference<TransportResponse> responseRef = new AtomicReference<>();
-        handler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onResponseSent(long requestId, String action, TransportResponse response) {
-                requestIdRef.set(requestId);
-                actionRef.set(action);
-                responseRef.set(response);
-            }
-        });
-        handler.sendResponse(version, Collections.emptySet(), channel, requestId, action, response, compress, isHandshake);
-
-        BytesReference reference = channel.getMessageCaptor().get();
-        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
-        if (randomBoolean()) {
-            sendListener.onResponse(null);
-        } else {
-            sendListener.onFailure(new IOException("failed"));
-        }
-        assertEquals(requestId, requestIdRef.get());
-        assertEquals(action, actionRef.get());
-        assertEquals(response, responseRef.get());
-
-        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
-        final Tuple<Header, BytesReference> tuple = message.get();
-        final Header header = tuple.v1();
-        final TestResponse message = new TestResponse(tuple.v2().streamInput());
-        assertEquals(version, header.getVersion());
-        assertEquals(requestId, header.getRequestId());
-        assertFalse(header.isRequest());
-        assertTrue(header.isResponse());
-        if (isHandshake) {
-            assertTrue(header.isHandshake());
-        } else {
-            assertFalse(header.isHandshake());
-        }
-        if (compress) {
-            assertTrue(header.isCompressed());
-        } else {
-            assertFalse(header.isCompressed());
-        }
-
-        assertFalse(header.isError());
-
-        assertEquals(value, message.value);
-        assertEquals("header_value", header.getHeaders().v1().get("header"));
-    }
-
-    public void testErrorResponse() throws IOException {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
-        String action = "handshake";
-        long requestId = randomLongBetween(0, 300);
-        threadContext.putHeader("header", "header_value");
-        OpenSearchException error = new OpenSearchException("boom");
-
-        AtomicLong requestIdRef = new AtomicLong();
-        AtomicReference<String> actionRef = new AtomicReference<>();
-        AtomicReference<Exception> responseRef = new AtomicReference<>();
-        handler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onResponseSent(long requestId, String action, Exception error) {
-                requestIdRef.set(requestId);
-                actionRef.set(action);
-                responseRef.set(error);
-            }
-        });
-        handler.sendErrorResponse(version, Collections.emptySet(), channel, requestId, action, error);
-
-        BytesReference reference = channel.getMessageCaptor().get();
-        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
-        if (randomBoolean()) {
-            sendListener.onResponse(null);
-        } else {
-            sendListener.onFailure(new IOException("failed"));
-        }
-        assertEquals(requestId, requestIdRef.get());
-        assertEquals(action, actionRef.get());
-        assertEquals(error, responseRef.get());
-
-        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
-        final Tuple<Header, BytesReference> tuple = message.get();
-        final Header header = tuple.v1();
-        assertEquals(version, header.getVersion());
-        assertEquals(requestId, header.getRequestId());
-        assertFalse(header.isRequest());
-        assertTrue(header.isResponse());
-        assertFalse(header.isCompressed());
-        assertFalse(header.isHandshake());
-        assertTrue(header.isError());
-
-        RemoteTransportException remoteException = tuple.v2().streamInput().readException();
-        assertThat(remoteException.getCause(), instanceOf(OpenSearchException.class));
-        assertEquals(remoteException.getCause().getMessage(), "boom");
-        assertEquals(action, remoteException.action());
-        assertEquals(channel.getLocalAddress(), remoteException.address().address());
-
-        assertEquals("header_value", header.getHeaders().v1().get("header"));
-    }
 }
diff --git a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java
index 7ab78cca7d615..7c5c9ec12360d 100644
--- a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java
+++ b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java
@@ -43,7 +43,6 @@
 import org.opensearch.common.network.NetworkService;
 import org.opensearch.common.network.NetworkUtils;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.MockPageCacheRecycler;
 import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
@@ -540,19 +539,7 @@ private void testExceptionHandling(
             final PlainActionFuture<Void> listener = new PlainActionFuture<>();
             channel.addCloseListener(listener);
 
-            TcpTransport.handleException(
-                channel,
-                exception,
-                lifecycle,
-                new OutboundHandler(
-                    randomAlphaOfLength(10),
-                    Version.CURRENT,
-                    new String[0],
-                    new StatsTracker(),
-                    testThreadPool,
-                    BigArrays.NON_RECYCLING_INSTANCE
-                )
-            );
+            TcpTransport.handleException(channel, exception, lifecycle, new OutboundHandler(new StatsTracker(), testThreadPool));
 
             if (expectClosed) {
                 assertTrue(listener.isDone());
diff --git a/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java b/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java
index 05296e9308657..cbd1b959f5f16 100644
--- a/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java
+++ b/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java
@@ -33,12 +33,6 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
-import org.opensearch.Version;
-import org.opensearch.action.admin.cluster.stats.ClusterStatsAction;
-import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest;
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.test.MockLogAppender;
 import org.opensearch.test.OpenSearchTestCase;
@@ -49,7 +43,7 @@
 import static org.mockito.Mockito.mock;
 
 @TestLogging(value = "org.opensearch.transport.TransportLogger:trace", reason = "to ensure we log network events on TRACE level")
-public class TransportLoggerTests extends OpenSearchTestCase {
+public abstract class TransportLoggerTests extends OpenSearchTestCase {
     public void testLoggingHandler() throws Exception {
         try (MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getLogger(TransportLogger.class))) {
             final String writePattern = ".*\\[length: \\d+"
@@ -90,20 +84,5 @@ public void testLoggingHandler() throws Exception {
         }
     }
 
-    private BytesReference buildRequest() throws IOException {
-        boolean compress = randomBoolean();
-        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
-            OutboundMessage.Request request = new OutboundMessage.Request(
-                new ThreadContext(Settings.EMPTY),
-                new String[0],
-                new ClusterStatsRequest(),
-                Version.CURRENT,
-                ClusterStatsAction.NAME,
-                randomInt(30),
-                false,
-                compress
-            );
-            return request.serialize(bytesStreamOutput);
-        }
-    }
+    public abstract BytesReference buildRequest() throws IOException;
 }
diff --git a/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStreamTests.java
similarity index 99%
rename from server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java
rename to server/src/test/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStreamTests.java
index 89018b7353e7c..eaa35469b9ec0 100644
--- a/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStreamTests.java
@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java
new file mode 100644
index 0000000000000..bd85939c753fa
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.InboundDecoderTests;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class NativeInboundDecoderTests extends InboundDecoderTests {
+
+    @Override
+    protected BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        Writeable transportMessage
+    ) throws IOException {
+        NativeOutboundMessage message;
+        if (isRequest) {
+            message = new NativeOutboundMessage.Request(
+                threadContext,
+                new String[0],
+                transportMessage,
+                version,
+                action,
+                requestId,
+                handshake,
+                compress
+            );
+        } else {
+            message = new NativeOutboundMessage.Response(
+                threadContext,
+                Collections.emptySet(),
+                transportMessage,
+                version,
+                requestId,
+                handshake,
+                compress
+            );
+        }
+
+        return message.serialize(new BytesStreamOutput());
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java
new file mode 100644
index 0000000000000..ec0c1a50d5560
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.InboundHandlerTests;
+
+import java.io.IOException;
+
+public class NativeInboundHandlerTests extends InboundHandlerTests {
+
+    @Override
+    public BytesReference serializeOutboundRequest(
+        ThreadContext threadContext,
+        Writeable message,
+        Version version,
+        String action,
+        long requestId,
+        boolean compress,
+        boolean handshake
+    ) throws IOException {
+        NativeOutboundMessage.Request request = new NativeOutboundMessage.Request(
+            threadContext,
+            new String[0],
+            message,
+            version,
+            action,
+            requestId,
+            handshake,
+            compress
+        );
+        return request.serialize(new BytesStreamOutput());
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java
new file mode 100644
index 0000000000000..6b5bf46eee3ae
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.transport.InboundPipelineTests;
+import org.opensearch.transport.TestRequest;
+import org.opensearch.transport.TestResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class NativeInboundPipelineTests extends InboundPipelineTests {
+
+    @Override
+    protected BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        String value
+    ) throws IOException {
+        NativeOutboundMessage message;
+        if (isRequest) {
+            message = new NativeOutboundMessage.Request(
+                threadContext,
+                new String[0],
+                new TestRequest(value),
+                version,
+                action,
+                requestId,
+                handshake,
+                compress
+            );
+        } else {
+            message = new NativeOutboundMessage.Response(
+                threadContext,
+                Collections.emptySet(),
+                new TestResponse(value),
+                version,
+                requestId,
+                handshake,
+                compress
+            );
+        }
+
+        return message.serialize(new BytesStreamOutput());
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java
new file mode 100644
index 0000000000000..75c4e84b4456e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java
@@ -0,0 +1,50 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.transport.TestRequest;
+
+import java.io.IOException;
+
+public class NativeOutboundMessageTests extends OpenSearchTestCase {
+
+    public void testNativeOutboundMessageRequestSerialization() throws IOException {
+        NativeOutboundMessage.Request message = new NativeOutboundMessage.Request(
+            new ThreadContext(Settings.EMPTY),
+            new String[0],
+            new TestRequest("content"),
+            Version.CURRENT,
+            "action",
+            1,
+            false,
+            false
+        );
+        BytesStreamOutput output = new BytesStreamOutput();
+        message.serialize(output);
+
+        BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes);
+        assertEquals(Version.CURRENT, input.getVersion());
+        // reading header details
+        assertEquals((byte) 'E', input.readByte());
+        assertEquals((byte) 'S', input.readByte());
+        assertNotEquals(0, input.readInt());
+        assertEquals(1, input.readLong());
+        assertEquals(0, input.readByte());
+        assertEquals(Version.CURRENT.id, input.readInt());
+        int variableHeaderSize = input.readInt();
+        assertNotEquals(-1, variableHeaderSize);
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java
new file mode 100644
index 0000000000000..db75d6ff45556
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.action.admin.cluster.stats.ClusterStatsAction;
+import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.test.junit.annotations.TestLogging;
+import org.opensearch.transport.TransportLoggerTests;
+
+import java.io.IOException;
+
+@TestLogging(value = "org.opensearch.transport.TransportLogger:trace", reason = "to ensure we log network events on TRACE level")
+public class NativeTransportLoggerTests extends TransportLoggerTests {
+
+    public BytesReference buildRequest() throws IOException {
+        boolean compress = randomBoolean();
+        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
+            NativeOutboundMessage.Request request = new NativeOutboundMessage.Request(
+                new ThreadContext(Settings.EMPTY),
+                new String[0],
+                new ClusterStatsRequest(),
+                Version.CURRENT,
+                ClusterStatsAction.NAME,
+                randomInt(30),
+                false,
+                compress
+            );
+            return request.serialize(new BytesStreamOutput());
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/transport/TestRequest.java b/test/framework/src/main/java/org/opensearch/transport/TestRequest.java
index 2fe917235e948..0bb5a6e16fff1 100644
--- a/test/framework/src/main/java/org/opensearch/transport/TestRequest.java
+++ b/test/framework/src/main/java/org/opensearch/transport/TestRequest.java
@@ -54,4 +54,8 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(value);
     }
+
+    public String getValue() {
+        return value;
+    }
 }
diff --git a/test/framework/src/main/java/org/opensearch/transport/TestResponse.java b/test/framework/src/main/java/org/opensearch/transport/TestResponse.java
index 14db8b3372bf2..7fd7c760c9cf6 100644
--- a/test/framework/src/main/java/org/opensearch/transport/TestResponse.java
+++ b/test/framework/src/main/java/org/opensearch/transport/TestResponse.java
@@ -54,4 +54,8 @@ public TestResponse(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(value);
     }
+
+    public String getValue() {
+        return value;
+    }
 }

From f8baa4ae79aa167004302eb09d39ad1c71cb64c8 Mon Sep 17 00:00:00 2001
From: Bhumika Saini <sabhumik@amazon.com>
Date: Tue, 14 May 2024 15:10:20 +0530
Subject: [PATCH 33/52] Allow clearing remote_store.compatibility_mode setting
 (#13646)

Signed-off-by: Bhumika Saini <sabhumik@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../RemoteStoreMigrationSettingsUpdateIT.java | 47 +++++++++++++------
 ...eMigrationShardAllocationBaseTestCase.java |  5 ++
 .../TransportClusterUpdateSettingsAction.java |  5 +-
 4 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b858ede4b78cf..3471cf545d4e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 - Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
+- Allow clearing `remote_store.compatibility_mode` setting ([#13646](https://github.com/opensearch-project/OpenSearch/pull/13646))
 
 ### Security
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
index b71f7d7cf7e4a..377bd9529ca7a 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
@@ -14,6 +14,7 @@
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Optional;
 
@@ -26,6 +27,8 @@
 public class RemoteStoreMigrationSettingsUpdateIT extends RemoteStoreMigrationShardAllocationBaseTestCase {
 
     private Client client;
+    private String nonRemoteNodeName;
+    private String remoteNodeName;
 
     // remote store backed index setting tests
 
@@ -120,18 +123,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
     // compatibility mode setting test
 
     public void testSwitchToStrictMode() throws Exception {
-        logger.info("Initialize cluster");
-        initializeCluster(false);
-
-        logger.info("Create a mixed mode cluster");
-        setClusterMode(MIXED.mode);
-        addRemote = true;
-        String remoteNodeName = internalCluster().startNode();
-        addRemote = false;
-        String nonRemoteNodeName = internalCluster().startNode();
-        internalCluster().validateClusterFormed();
-        assertNodeInCluster(remoteNodeName);
-        assertNodeInCluster(nonRemoteNodeName);
+        createMixedModeCluster();
 
         logger.info("Attempt switching to strict mode");
         SettingsException exception = assertThrows(SettingsException.class, () -> setClusterMode(STRICT.mode));
@@ -140,12 +132,39 @@ public void testSwitchToStrictMode() throws Exception {
             exception.getMessage()
         );
 
+        stopRemoteNode();
+
+        logger.info("Attempt switching to strict mode");
+        setClusterMode(STRICT.mode);
+    }
+
+    public void testClearCompatibilityModeSetting() throws Exception {
+        createMixedModeCluster();
+        stopRemoteNode();
+
+        logger.info("Attempt clearing compatibility mode");
+        clearClusterMode();
+    }
+
+    private void stopRemoteNode() throws IOException {
         logger.info("Stop remote node so that cluster had only non-remote nodes");
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(remoteNodeName));
         ensureStableCluster(2);
+    }
 
-        logger.info("Attempt switching to strict mode");
-        setClusterMode(STRICT.mode);
+    private void createMixedModeCluster() {
+        logger.info("Initialize cluster");
+        initializeCluster(false);
+
+        logger.info("Create a mixed mode cluster");
+        setClusterMode(MIXED.mode);
+        addRemote = true;
+        remoteNodeName = internalCluster().startNode();
+        addRemote = false;
+        nonRemoteNodeName = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+        assertNodeInCluster(remoteNodeName);
+        assertNodeInCluster(nonRemoteNodeName);
     }
 
     // bootstrap a cluster
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
index cf689aa554c8b..3591846a93bc8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
@@ -55,6 +55,11 @@ protected void setClusterMode(String mode) {
         assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
     }
 
+    protected void clearClusterMode() {
+        updateSettingsRequest.persistentSettings(Settings.builder().putNull(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey()));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+    }
+
     // set the migration direction for cluster [remote_store, docrep, none]
     protected void setDirection(String direction) {
         updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), direction));
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index 6292d32fee26d..216e1fb2ed1cc 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -66,7 +66,6 @@
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -285,9 +284,9 @@ public ClusterState execute(final ClusterState currentState) {
     public void validateCompatibilityModeSettingRequest(ClusterUpdateSettingsRequest request, ClusterState clusterState) {
         Settings settings = Settings.builder().put(request.persistentSettings()).put(request.transientSettings()).build();
         if (RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING.exists(settings)) {
-            String value = settings.get(RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey()).toLowerCase(Locale.ROOT);
+            String value = RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(settings).mode;
             validateAllNodesOfSameVersion(clusterState.nodes());
-            if (value.equals(RemoteStoreNodeService.CompatibilityMode.STRICT.mode)) {
+            if (RemoteStoreNodeService.CompatibilityMode.STRICT.mode.equals(value)) {
                 validateAllNodesOfSameType(clusterState.nodes());
                 validateIndexSettings(clusterState);
             }

From a03db0dcd304a600463f665a2bd9e43364985153 Mon Sep 17 00:00:00 2001
From: chengwushi-netapp <153049940+chengwushi-netapp@users.noreply.github.com>
Date: Wed, 15 May 2024 00:21:47 +1000
Subject: [PATCH 34/52] Main repository azure managed identity support (#12559)

* Added support for Azure Managed Identity in repository-azure

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* Refactor tokenCredentialType as an enum when constructing AzureStorageSetting

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* fixed indentation

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* fixed syntax

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* removed unused imports

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* applied changes after running :plugins:repository-azure:spotlessApply

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* added transitive dependencies

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* changed getStorageBlobEndpoint to private and using asm version from buildSrc/version.properties

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* run spotlessApply

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* update shas for asm 9.7

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* use version.jna for jna-platform

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* change string 'core.windows.net' to be a constant, use uri.create instead of new uri, and added a few comments for clarity

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* added one more comment line

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* refactor TokenCredentialType to not have NOT_APPLICABLE

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* refactored code based on recommended changes from Andriy Redko

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* added a jvm security policy for reactor-core jar

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* fixed failing forbidden api fix

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* removed the jvm security policy for reactor-core which was added

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* refactored code, such that storage endpoint is not evaluated at compiled time

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* refactored token credential types checks

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* changed clientlogger in azurestoragesetting to 'AzureStorageService'

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* added a nullable argument to getStorageEndpoint

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

* Fix IdentityClient security permissions, get rid of connection string (since it is not applicable to managed identity configuration)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Responded to feedback from Andrew Ross, fixed typo, spelling, added shudownNow() and Thread.currentThread().interrupt()

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>

---------

Signed-off-by: Chengwu Shi <chengwu.shi@netapp.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: Andriy Redko <andriy.redko@aiven.io>
---
 CHANGELOG.md                                  |   1 +
 .../gradle/precommit/LicenseAnalyzer.java     |   2 +-
 plugins/repository-azure/build.gradle         |  86 ++-
 .../licenses/accessors-smart-2.5.0.jar.sha1   |   1 +
 .../licenses/accessors-smart-LICENSE.txt      | 202 ++++++
 .../licenses/accessors-smart-NOTICE.txt       |   0
 .../licenses/asm-9.7.jar.sha1                 |   1 +
 .../repository-azure/licenses/asm-LICENSE.txt |  27 +
 .../repository-azure/licenses/asm-NOTICE.txt  |   0
 .../licenses/azure-identity-1.11.4.jar.sha1   |   1 +
 .../licenses/content-type-2.3.jar.sha1        |   1 +
 .../licenses/content-type-LICENSE.txt         | 202 ++++++
 .../licenses/content-type-NOTICE.txt          |   0
 .../licenses/jna-platform-5.13.0.jar.sha1     |   1 +
 .../licenses/jna-platform-LICENSE.txt         |  26 +
 .../licenses/jna-platform-NOTICE.txt          |   0
 .../licenses/json-smart-2.5.0.jar.sha1        |   1 +
 .../licenses/json-smart-LICENSE.txt           | 202 ++++++
 .../licenses/json-smart-NOTICE.txt            |   0
 .../licenses/lang-tag-1.7.jar.sha1            |   1 +
 .../licenses/lang-tag-LICENSE.txt             | 202 ++++++
 .../licenses/lang-tag-NOTICE.txt              |   0
 .../licenses/msal4j-1.14.3.jar.sha1           |   1 +
 .../licenses/msal4j-LICENSE.txt               |  21 +
 .../licenses/msal4j-NOTICE.txt                |   0
 ...sal4j-persistence-extension-1.2.0.jar.sha1 |   1 +
 .../msal4j-persistence-extension-LICENSE.txt  |  21 +
 .../msal4j-persistence-extension-NOTICE.txt   |   0
 .../licenses/nimbus-jose-jwt-9.37.3.jar.sha1  |   1 +
 .../licenses/nimbus-jose-jwt-LICENSE.txt      | 202 ++++++
 .../licenses/nimbus-jose-jwt-NOTICE.txt       |   0
 .../licenses/oauth2-oidc-sdk-11.9.1.jar.sha1  |   1 +
 .../licenses/oauth2-oidc-sdk-LICENSE.txt      | 202 ++++++
 .../licenses/oauth2-oidc-sdk-NOTICE.txt       |   0
 .../repositories/azure/AzureBlobStore.java    |   2 +-
 .../azure/AzureRepositoryPlugin.java          |   1 +
 .../azure/AzureStorageService.java            |  61 +-
 .../azure/AzureStorageSettings.java           |  91 ++-
 .../azure/TokenCredentialType.java            |  40 ++
 .../plugin-metadata/plugin-security.policy    |   3 +
 .../azure/AzureStorageServiceTests.java       | 573 +++++++++++++-----
 .../org/opensearch/bootstrap/security.policy  |   5 +
 42 files changed, 2028 insertions(+), 155 deletions(-)
 create mode 100644 plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/accessors-smart-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/accessors-smart-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/asm-9.7.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/asm-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/asm-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/content-type-2.3.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/content-type-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/content-type-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/jna-platform-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/jna-platform-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/json-smart-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/json-smart-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/lang-tag-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/lang-tag-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/msal4j-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/msal4j-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt
 create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt
 create mode 100644 plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3471cf545d4e2..65c5fa1223cc6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423))
 - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
 - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
 
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java
index 4c63516126566..c3acd12e5a1cf 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java
@@ -145,7 +145,7 @@ public class LicenseAnalyzer {
                     + "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n"
                     + "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n"
                     + "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n"
-                    + "SOFTWARE\\.\n").replaceAll("\\s+", "\\\\s*"),
+                    + "SOFTWARE\\.?\n").replaceAll("\\s+", "\\\\s*"),
                 Pattern.DOTALL
             )
         ),
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index c7836170d658f..ff62c328c7e74 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -56,6 +56,21 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
+  api 'com.azure:azure-identity:1.11.4'
+  // Start of transitive dependencies for azure-identity
+  api 'com.microsoft.azure:msal4j-persistence-extension:1.2.0'
+  api "net.java.dev.jna:jna-platform:${versions.jna}"
+  api 'com.microsoft.azure:msal4j:1.14.3'
+  api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
+  api 'com.nimbusds:nimbus-jose-jwt:9.37.3'
+  api 'com.nimbusds:content-type:2.3'
+  api 'com.nimbusds:lang-tag:1.7'
+  // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,
+  // selected the higher version which is 2.5.0
+  api 'net.minidev:json-smart:2.5.0'
+  api 'net.minidev:accessors-smart:2.5.0'
+  api "org.ow2.asm:asm:${versions.asm}"
+  // End of transitive dependencies for azure-identity
   api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
   api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
@@ -180,7 +195,76 @@ thirdPartyAudit {
     'io.micrometer.observation.ObservationHandler',
     'io.micrometer.observation.ObservationRegistry',
     'io.micrometer.observation.ObservationRegistry$ObservationConfig',
-    'io.micrometer.tracing.handler.DefaultTracingObservationHandler'
+    'io.micrometer.tracing.handler.DefaultTracingObservationHandler',
+    // Start of the list of classes from the optional compile/provided dependencies used in "com.nimbusds:oauth2-oidc-sdk".
+    'com.google.crypto.tink.subtle.Ed25519Sign',
+    'com.google.crypto.tink.subtle.Ed25519Sign$KeyPair',
+    'com.google.crypto.tink.subtle.Ed25519Verify',
+    'com.google.crypto.tink.subtle.X25519',
+    'com.google.crypto.tink.subtle.XChaCha20Poly1305',
+    'jakarta.servlet.ServletRequest',
+    'jakarta.servlet.http.HttpServletRequest',
+    'jakarta.servlet.http.HttpServletResponse',
+    'javax.servlet.ServletRequest',
+    'javax.servlet.http.HttpServletRequest',
+    'javax.servlet.http.HttpServletResponse',
+    // net.shibboleth.utilities:java-support.* is declared as optional in the plugin `bnd-maven-plugin` used in "com.nimbusds:oauth2-oidc-sdk"
+    // Worth nothing that, the latest dependency "net.shibboleth.utilities:java-support:8.0.0" has many vulnerabilities.
+    // Hence ignored.
+    'net.shibboleth.utilities.java.support.xml.SerializeSupport',
+    'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+    'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+    'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+    'org.bouncycastle.cert.X509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+    'org.bouncycastle.crypto.InvalidCipherTextException',
+    'org.bouncycastle.crypto.engines.AESEngine',
+    'org.bouncycastle.crypto.modes.GCMBlockCipher',
+    'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
+    'org.bouncycastle.jce.provider.BouncyCastleProvider',
+    'org.bouncycastle.openssl.PEMKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.cryptomator.siv.SivMode',
+    'org.opensaml.core.config.InitializationException',
+    'org.opensaml.core.config.InitializationService',
+    'org.opensaml.core.xml.XMLObject',
+    'org.opensaml.core.xml.XMLObjectBuilder',
+    'org.opensaml.core.xml.XMLObjectBuilderFactory',
+    'org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport',
+    'org.opensaml.core.xml.io.Marshaller',
+    'org.opensaml.core.xml.io.MarshallerFactory',
+    'org.opensaml.core.xml.io.MarshallingException',
+    'org.opensaml.core.xml.io.Unmarshaller',
+    'org.opensaml.core.xml.io.UnmarshallerFactory',
+    'org.opensaml.core.xml.schema.XSString',
+    'org.opensaml.core.xml.schema.impl.XSStringBuilder',
+    'org.opensaml.saml.saml2.core.Assertion',
+    'org.opensaml.saml.saml2.core.Attribute',
+    'org.opensaml.saml.saml2.core.AttributeStatement',
+    'org.opensaml.saml.saml2.core.AttributeValue',
+    'org.opensaml.saml.saml2.core.Audience',
+    'org.opensaml.saml.saml2.core.AudienceRestriction',
+    'org.opensaml.saml.saml2.core.AuthnContext',
+    'org.opensaml.saml.saml2.core.AuthnContextClassRef',
+    'org.opensaml.saml.saml2.core.AuthnStatement',
+    'org.opensaml.saml.saml2.core.Conditions',
+    'org.opensaml.saml.saml2.core.Issuer',
+    'org.opensaml.saml.saml2.core.NameID',
+    'org.opensaml.saml.saml2.core.Subject',
+    'org.opensaml.saml.saml2.core.SubjectConfirmation',
+    'org.opensaml.saml.saml2.core.SubjectConfirmationData',
+    'org.opensaml.saml.security.impl.SAMLSignatureProfileValidator',
+    'org.opensaml.security.credential.BasicCredential',
+    'org.opensaml.security.credential.Credential',
+    'org.opensaml.security.credential.UsageType',
+    'org.opensaml.xmlsec.signature.Signature',
+    'org.opensaml.xmlsec.signature.support.SignatureException',
+    'org.opensaml.xmlsec.signature.support.SignatureValidator',
+    'org.opensaml.xmlsec.signature.support.Signer',
+    // End of the list of classes from the optional compile/provided dependencies used in "com.nimbusds:oauth2-oidc-sdk".
   )
 
   ignoreViolations(
diff --git a/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
new file mode 100644
index 0000000000000..1578c94fcdc7b
--- /dev/null
+++ b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
@@ -0,0 +1 @@
+aca011492dfe9c26f4e0659028a4fe0970829dd8
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt b/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/accessors-smart-NOTICE.txt b/plugins/repository-azure/licenses/accessors-smart-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/asm-9.7.jar.sha1 b/plugins/repository-azure/licenses/asm-9.7.jar.sha1
new file mode 100644
index 0000000000000..84c9a9703af6d
--- /dev/null
+++ b/plugins/repository-azure/licenses/asm-9.7.jar.sha1
@@ -0,0 +1 @@
+073d7b3086e14beb604ced229c302feff6449723
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/asm-LICENSE.txt b/plugins/repository-azure/licenses/asm-LICENSE.txt
new file mode 100644
index 0000000000000..c71bb7bac5d4d
--- /dev/null
+++ b/plugins/repository-azure/licenses/asm-LICENSE.txt
@@ -0,0 +1,27 @@
+ASM: a very small and fast Java bytecode manipulation framework
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/plugins/repository-azure/licenses/asm-NOTICE.txt b/plugins/repository-azure/licenses/asm-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
new file mode 100644
index 0000000000000..c8d98ba9c8ad2
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
@@ -0,0 +1 @@
+59b5ce48888f638b80d85ef5aa0e22a265d3dc89
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/content-type-2.3.jar.sha1 b/plugins/repository-azure/licenses/content-type-2.3.jar.sha1
new file mode 100644
index 0000000000000..e18bbaec9a89c
--- /dev/null
+++ b/plugins/repository-azure/licenses/content-type-2.3.jar.sha1
@@ -0,0 +1 @@
+e3aa0be212d7a42839a8f3f506f5b990bcce0222
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/content-type-LICENSE.txt b/plugins/repository-azure/licenses/content-type-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/content-type-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/content-type-NOTICE.txt b/plugins/repository-azure/licenses/content-type-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1 b/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1
new file mode 100644
index 0000000000000..e2a8ba1c1bbd3
--- /dev/null
+++ b/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1
@@ -0,0 +1 @@
+88e9a306715e9379f3122415ef4ae759a352640d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jna-platform-LICENSE.txt b/plugins/repository-azure/licenses/jna-platform-LICENSE.txt
new file mode 100644
index 0000000000000..c5a025f0c3e6d
--- /dev/null
+++ b/plugins/repository-azure/licenses/jna-platform-LICENSE.txt
@@ -0,0 +1,26 @@
+SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1
+
+Java Native Access (JNA) is licensed under the LGPL, version 2.1
+or later, or (from version 4.0 onward) the Apache License,
+version 2.0.
+
+You can freely decide which license you want to apply to the project.
+
+You may obtain a copy of the LGPL License at:
+
+http://www.gnu.org/licenses/licenses.html
+
+A copy is also included in the downloadable source code package
+containing JNA, in file "LGPL2.1", under the same directory
+as this file.
+
+You may obtain a copy of the Apache License at:
+
+http://www.apache.org/licenses/
+
+A copy is also included in the downloadable source code package
+containing JNA, in file "AL2.0", under the same directory
+as this file.
+
+Commercial support may be available, please e-mail
+twall[at]users[dot]sf[dot]net.
diff --git a/plugins/repository-azure/licenses/jna-platform-NOTICE.txt b/plugins/repository-azure/licenses/jna-platform-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
new file mode 100644
index 0000000000000..3ec055efa1255
--- /dev/null
+++ b/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
@@ -0,0 +1 @@
+57a64f421b472849c40e77d2e7cce3a141b41e99
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/json-smart-LICENSE.txt b/plugins/repository-azure/licenses/json-smart-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/json-smart-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/json-smart-NOTICE.txt b/plugins/repository-azure/licenses/json-smart-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1 b/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1
new file mode 100644
index 0000000000000..9cd79d1dba715
--- /dev/null
+++ b/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1
@@ -0,0 +1 @@
+97c73ecd70bc7e8eefb26c5eea84f251a63f1031
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/lang-tag-LICENSE.txt b/plugins/repository-azure/licenses/lang-tag-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/lang-tag-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/lang-tag-NOTICE.txt b/plugins/repository-azure/licenses/lang-tag-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
new file mode 100644
index 0000000000000..2a6e42e3f2b48
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
@@ -0,0 +1 @@
+117b28c41bd760f979ed1b6467c5ec491f0d4d60
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-LICENSE.txt b/plugins/repository-azure/licenses/msal4j-LICENSE.txt
new file mode 100644
index 0000000000000..21071075c2459
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-LICENSE.txt
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/plugins/repository-azure/licenses/msal4j-NOTICE.txt b/plugins/repository-azure/licenses/msal4j-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
new file mode 100644
index 0000000000000..cfcf7548b7694
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
@@ -0,0 +1 @@
+1111a95878de8745ddc9de132df18ebd9ca7024d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt b/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
new file mode 100644
index 0000000000000..21071075c2459
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt b/plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
new file mode 100644
index 0000000000000..7278cd8994f71
--- /dev/null
+++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
@@ -0,0 +1 @@
+700f71ffefd60c16bd8ce711a956967ea9071cec
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt b/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt b/plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
new file mode 100644
index 0000000000000..96d9a196a172a
--- /dev/null
+++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
@@ -0,0 +1 @@
+fa9a2e447e2cef4dfda40a854dd7ec35624a7799
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt b/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt b/plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java
index e76a6bdd16764..acaaa043df3ac 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java
@@ -175,7 +175,7 @@ public BlobContainer blobContainer(BlobPath path) {
     }
 
     @Override
-    public void close() {
+    public void close() throws IOException {
         service.close();
     }
 
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java
index 78db7cb2d0ea7..aca213f9fed79 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java
@@ -91,6 +91,7 @@ public List<Setting<?>> getSettings() {
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
             AzureStorageSettings.SAS_TOKEN_SETTING,
+            AzureStorageSettings.TOKEN_CREDENTIAL_TYPE_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
             AzureStorageSettings.MAX_RETRIES_SETTING,
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
index 74edd4f3eb23c..f39ed185d8b35 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
@@ -47,7 +47,6 @@
 import com.azure.storage.blob.BlobServiceClientBuilder;
 import com.azure.storage.blob.models.ParallelTransferOptions;
 import com.azure.storage.blob.specialized.BlockBlobAsyncClient;
-import com.azure.storage.common.implementation.connectionstring.StorageConnectionString;
 import com.azure.storage.common.implementation.connectionstring.StorageEndpoint;
 import com.azure.storage.common.policy.RequestRetryOptions;
 import com.azure.storage.common.policy.RetryPolicyType;
@@ -59,14 +58,19 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
+import java.io.IOException;
 import java.net.Authenticator;
 import java.net.PasswordAuthentication;
 import java.net.URISyntaxException;
+import java.security.AccessController;
 import java.security.InvalidKeyException;
+import java.security.PrivilegedAction;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -99,6 +103,37 @@ public class AzureStorageService implements AutoCloseable {
     // 'package' for testing
     volatile Map<String, AzureStorageSettings> storageSettings = emptyMap();
     private final Map<AzureStorageSettings, ClientState> clients = new ConcurrentHashMap<>();
+    private final ExecutorService executor;
+
+    private static final class IdentityClientThreadFactory implements ThreadFactory {
+        final ThreadGroup group;
+        final AtomicInteger threadNumber = new AtomicInteger(1);
+        final String namePrefix;
+
+        @SuppressWarnings("removal")
+        IdentityClientThreadFactory(String namePrefix) {
+            this.namePrefix = namePrefix;
+            SecurityManager s = System.getSecurityManager();
+            group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
+        }
+
+        @Override
+        public Thread newThread(Runnable r) {
+            Thread t = new Thread(group, new Runnable() {
+                @SuppressWarnings("removal")
+                public void run() {
+                    AccessController.doPrivileged(new PrivilegedAction<>() {
+                        public Void run() {
+                            r.run();
+                            return null;
+                        }
+                    });
+                }
+            }, namePrefix + "[T#" + threadNumber.getAndIncrement() + "]", 0);
+            t.setDaemon(true);
+            return t;
+        }
+    }
 
     static {
         // See please:
@@ -112,6 +147,9 @@ public AzureStorageService(Settings settings) {
         // eagerly load client settings so that secure settings are read
         final Map<String, AzureStorageSettings> clientsSettings = AzureStorageSettings.load(settings);
         refreshAndClearCache(clientsSettings);
+        executor = SocketAccess.doPrivilegedException(
+            () -> Executors.newCachedThreadPool(new IdentityClientThreadFactory("azure-identity-client"))
+        );
     }
 
     /**
@@ -162,7 +200,6 @@ public Tuple<BlobServiceClient, Supplier<Context>> client(String clientName, BiC
     private ClientState buildClient(AzureStorageSettings azureStorageSettings, BiConsumer<HttpRequest, HttpResponse> statsCollector)
         throws InvalidKeyException, URISyntaxException {
         final BlobServiceClientBuilder builder = createClientBuilder(azureStorageSettings);
-
         final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(new NioThreadFactory());
         final NettyAsyncHttpClientBuilder clientBuilder = new NettyAsyncHttpClientBuilder().eventLoopGroup(eventLoopGroup);
 
@@ -216,8 +253,7 @@ protected PasswordAuthentication getPasswordAuthentication() {
      * <a href="https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/storage/azure-storage-blob/migrationGuides/V8_V12.md#miscellaneous">migration guide</a> for mode details:
      */
     private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) {
-        final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger);
-        final StorageEndpoint endpoint = storageConnectionString.getBlobEndpoint();
+        final StorageEndpoint endpoint = settings.getStorageEndpoint(logger);
 
         if (endpoint == null || endpoint.getPrimaryUri() == null) {
             throw new IllegalArgumentException("connectionString missing required settings to derive blob service primary endpoint.");
@@ -247,9 +283,8 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde
         return builder;
     }
 
-    private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException,
-        URISyntaxException {
-        return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().connectionString(settings.getConnectString()));
+    private BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException {
+        return SocketAccess.doPrivilegedException(() -> settings.configure(new BlobServiceClientBuilder(), executor, logger));
     }
 
     /**
@@ -295,9 +330,19 @@ public Map<String, AzureStorageSettings> refreshAndClearCache(Map<String, AzureS
     }
 
     @Override
-    public void close() {
+    public void close() throws IOException {
         this.clients.values().forEach(this::closeInternally);
         this.clients.clear();
+        this.executor.shutdown();
+        try {
+            if (this.executor.awaitTermination(30, TimeUnit.SECONDS) == false) {
+                this.executor.shutdownNow();
+                logger.warning("The executor was not shutdown gracefully with 30 seconds");
+            }
+        } catch (final InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            throw new IOException(ex);
+        }
     }
 
     public Duration getBlobRequestTimeout(String clientName) {
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java
index e73ded679cf2b..1e09b89b8dadf 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java
@@ -32,7 +32,16 @@
 
 package org.opensearch.repositories.azure;
 
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.identity.ManagedIdentityCredential;
+import com.azure.identity.ManagedIdentityCredentialBuilder;
+import com.azure.identity.implementation.CredentialBuilderBaseHelper;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.implementation.Constants;
+import com.azure.storage.common.implementation.connectionstring.StorageConnectionString;
+import com.azure.storage.common.implementation.connectionstring.StorageEndpoint;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.TriFunction;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.common.settings.SecureSetting;
 import org.opensearch.common.settings.Setting;
@@ -45,11 +54,14 @@
 import org.opensearch.core.common.settings.SecureString;
 
 import java.net.InetAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Function;
 
 final class AzureStorageSettings {
 
@@ -77,6 +89,18 @@ final class AzureStorageSettings {
         key -> SecureSetting.secureString(key, null)
     );
 
+    /** Azure token credentials such as Managed Identity */
+    public static final AffixSetting<String> TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting(
+        AZURE_CLIENT_PREFIX_KEY,
+        "token_credential_type",
+        key -> Setting.simpleString(key, value -> {
+            if (Strings.hasText(value) == true) {
+                TokenCredentialType.valueOfType(value);
+            }
+        }, Property.NodeScope),
+        () -> ACCOUNT_SETTING
+    );
+
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
     public static final AffixSetting<Integer> MAX_RETRIES_SETTING = Setting.affixKeySetting(
         AZURE_CLIENT_PREFIX_KEY,
@@ -194,7 +218,9 @@ final class AzureStorageSettings {
     );
 
     private final String account;
-    private final String connectString;
+    private final String tokenCredentialType;
+    private final TriFunction<BlobServiceClientBuilder, ExecutorService, ClientLogger, BlobServiceClientBuilder> clientBuilder;
+    private final Function<ClientLogger, StorageEndpoint> endpointBuilder;
     private final String endpointSuffix;
     private final TimeValue timeout;
     private final int maxRetries;
@@ -208,7 +234,9 @@ final class AzureStorageSettings {
     // copy-constructor
     private AzureStorageSettings(
         String account,
-        String connectString,
+        String tokenCredentialType,
+        TriFunction<BlobServiceClientBuilder, ExecutorService, ClientLogger, BlobServiceClientBuilder> clientBuilder,
+        Function<ClientLogger, StorageEndpoint> endpointBuilder,
         String endpointSuffix,
         TimeValue timeout,
         int maxRetries,
@@ -220,7 +248,9 @@ private AzureStorageSettings(
         ProxySettings proxySettings
     ) {
         this.account = account;
-        this.connectString = connectString;
+        this.tokenCredentialType = tokenCredentialType;
+        this.clientBuilder = clientBuilder;
+        this.endpointBuilder = endpointBuilder;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -236,6 +266,7 @@ private AzureStorageSettings(
         String account,
         String key,
         String sasToken,
+        String tokenCredentialType,
         String endpointSuffix,
         TimeValue timeout,
         int maxRetries,
@@ -246,7 +277,37 @@ private AzureStorageSettings(
         ProxySettings proxySettings
     ) {
         this.account = account;
-        this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
+        this.tokenCredentialType = tokenCredentialType;
+        if (Strings.hasText(tokenCredentialType) == true) {
+            this.endpointBuilder = (logger) -> {
+                String tokenCredentialEndpointSuffix = endpointSuffix;
+                if (Strings.hasText(tokenCredentialEndpointSuffix) == false) {
+                    // Default to "core.windows.net".
+                    tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS;
+                }
+                final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix);
+                final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix);
+                return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint);
+            };
+
+            this.clientBuilder = (builder, executor, logger) -> builder.credential(new ManagedIdentityCredentialBuilder() {
+                @Override
+                public ManagedIdentityCredential build() {
+                    // Use the privileged executor with IdentityClient instance
+                    CredentialBuilderBaseHelper.getClientOptions(this).setExecutorService(executor);
+                    return super.build();
+                }
+            }.build()).endpoint(endpointBuilder.apply(logger).getPrimaryUri());
+        } else {
+            final String connectString = buildConnectString(account, key, sasToken, endpointSuffix);
+
+            this.endpointBuilder = (logger) -> {
+                final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectString, logger);
+                return storageConnectionString.getBlobEndpoint();
+            };
+
+            this.clientBuilder = (builder, executor, logger) -> builder.connectionString(connectString);
+        }
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -258,6 +319,14 @@ private AzureStorageSettings(
         this.proxySettings = proxySettings;
     }
 
+    public String getTokenCredentialType() {
+        return tokenCredentialType;
+    }
+
+    public StorageEndpoint getStorageEndpoint(ClientLogger logger) {
+        return endpointBuilder.apply(logger);
+    }
+
     public String getEndpointSuffix() {
         return endpointSuffix;
     }
@@ -274,10 +343,6 @@ public ProxySettings getProxySettings() {
         return proxySettings;
     }
 
-    public String getConnectString() {
-        return connectString;
-    }
-
     private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
         final boolean hasSasToken = Strings.hasText(sasToken);
         final boolean hasKey = Strings.hasText(key);
@@ -325,6 +390,7 @@ public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
         sb.append("account='").append(account).append('\'');
         sb.append(", timeout=").append(timeout);
+        sb.append(", tokenCredentialType=").append(tokenCredentialType).append('\'');
         sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
         sb.append(", maxRetries=").append(maxRetries);
         sb.append(", proxySettings=").append(proxySettings != ProxySettings.NO_PROXY_SETTINGS ? "PROXY_SET" : "PROXY_NOT_SET");
@@ -370,6 +436,7 @@ private static AzureStorageSettings getClientSettings(Settings settings, String
                 account.toString(),
                 key.toString(),
                 sasToken.toString(),
+                getValue(settings, clientName, TOKEN_CREDENTIAL_TYPE_SETTING),
                 getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING),
@@ -430,7 +497,9 @@ static Map<String, AzureStorageSettings> overrideLocationMode(
                 entry.getKey(),
                 new AzureStorageSettings(
                     entry.getValue().account,
-                    entry.getValue().connectString,
+                    entry.getValue().tokenCredentialType,
+                    entry.getValue().clientBuilder,
+                    entry.getValue().endpointBuilder,
                     entry.getValue().endpointSuffix,
                     entry.getValue().timeout,
                     entry.getValue().maxRetries,
@@ -445,4 +514,8 @@ static Map<String, AzureStorageSettings> overrideLocationMode(
         }
         return mapBuilder.immutableMap();
     }
+
+    public BlobServiceClientBuilder configure(BlobServiceClientBuilder builder, ExecutorService executor, ClientLogger logger) {
+        return clientBuilder.apply(builder, executor, logger);
+    }
 }
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java
new file mode 100644
index 0000000000000..1f78f73934231
--- /dev/null
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.azure;
+
+import java.util.Arrays;
+
+// Type of token credentials that the plugin supports
+public enum TokenCredentialType {
+    MANAGED_IDENTITY("managed");
+
+    private final String type;
+
+    TokenCredentialType(String type) {
+        this.type = type;
+    }
+
+    public static String[] getTokenCredentialTypes() {
+        return Arrays.stream(TokenCredentialType.values()).map(tokenCredentialType -> tokenCredentialType.type).toArray(String[]::new);
+    }
+
+    static TokenCredentialType valueOfType(String type) {
+        for (TokenCredentialType value : values()) {
+            if (value.type.equalsIgnoreCase(type) || value.name().equalsIgnoreCase(type)) {
+                return value;
+            }
+        }
+        throw new IllegalArgumentException(
+            "The token credential type '"
+                + type
+                + "' is unsupported, please use one of the following values: "
+                + String.join(", ", getTokenCredentialTypes())
+        );
+    }
+}
diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
index f3bf52ea46505..e8fbe35ebab1d 100644
--- a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
@@ -41,4 +41,7 @@ grant {
 
   // azure client set Authenticator for proxy username/password
   permission java.net.NetPermission "setDefaultAuthenticator";
+
+  // azure identity
+  permission java.util.PropertyPermission "os.name", "read";
 };
diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
index bb0eafc7d1d4a..ea74a49e593cf 100644
--- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
@@ -33,6 +33,7 @@
 package org.opensearch.repositories.azure;
 
 import com.azure.core.http.policy.HttpPipelinePolicy;
+import com.azure.identity.CredentialUnavailableException;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.common.policy.RequestRetryPolicy;
 import org.opensearch.common.settings.MockSecureSettings;
@@ -50,7 +51,6 @@
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Base64;
@@ -106,11 +106,105 @@ public void testCreateClientWithEndpointSuffix() throws IOException {
             .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix")
             .build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
-            assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
-            final BlobServiceClient client2 = azureStorageService.client("azure2").v1();
-            assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+                assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
+                final BlobServiceClient client2 = azureStorageService.client("azure2").v1();
+                assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
+            }
+        }
+    }
+
+    public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure clients without account key and sas token.
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            // Enabled managed identity for all clients
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .put("azure.client.azure2.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            // Defined an endpoint suffix for azure client 1 only.
+            .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix")
+            .build();
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                // Expect azure client 1 to use the custom endpoint suffix
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+                assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
+                // Expect azure client 2 to use the default endpoint suffix
+                final BlobServiceClient client2 = azureStorageService.client("azure2").v1();
+                assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
+            }
+        }
+    }
+
+    public void testCreateClientWithInvalidEndpointSuffix() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.key", encodeKey("mykey12"));
+        secureSettings.setString("azure.client.azure3.account", "myaccount1");
+        secureSettings.setString("azure.client.azure3.sas_token", encodeKey("mysastoken"));
+
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .put("azure.client.azure1.endpoint_suffix", "invalid endpoint suffix")
+            .put("azure.client.azure2.endpoint_suffix", "invalid endpoint suffix")
+            .put("azure.client.azure3.endpoint_suffix", "invalid endpoint suffix")
+            .build();
+
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                // Expect all clients 1 to fail due to invalid endpoint suffix
+                expectThrows(SettingsException.class, () -> azureStorageService.client("azure1").v1());
+                expectThrows(RuntimeException.class, () -> azureStorageService.client("azure2").v1());
+                expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1());
+            }
+        }
+    }
+
+    public void testGettingSecondaryStorageBlobEndpoint() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            // Enabled managed identity
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final Map<String, AzureStorageSettings> prevSettings = azureStorageService.refreshAndClearCache(Collections.emptyMap());
+                final Map<String, AzureStorageSettings> newSettings = AzureStorageSettings.overrideLocationMode(
+                    prevSettings,
+                    LocationMode.SECONDARY_ONLY
+                );
+                azureStorageService.refreshAndClearCache(newSettings);
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+                assertThat(client1.getAccountUrl(), equalTo("https://myaccount1-secondary.blob.core.windows.net"));
+            }
+        }
+    }
+
+    public void testClientUsingManagedIdentity() throws IOException {
+        // Enabled managed identity
+        final Settings settings = Settings.builder()
+            .setSecureSettings(buildSecureSettings())
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+
+                // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is
+                // not
+                // setup in the test
+                final CredentialUnavailableException e = expectThrows(CredentialUnavailableException.class, () -> client1.getAccountInfo());
+                assertThat(e.getMessage(), is("Managed Identity authentication is not available."));
+            }
         }
     }
 
@@ -128,29 +222,30 @@ public void testReinitClientSettings() throws IOException {
         secureSettings2.setString("azure.client.azure3.key", encodeKey("mykey23"));
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
-            final BlobServiceClient client12 = azureStorageService.client("azure2").v1();
-            assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
-            // client 3 is missing
-            final SettingsException e1 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure3"));
-            assertThat(e1.getMessage(), is("Unable to find client with name [azure3]"));
-            // update client settings
-            plugin.reload(settings2);
-            // old client 1 not changed
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
-            // new client 1 is changed
-            final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
-            assertThat(client21.getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net"));
-            // old client 2 not changed
-            assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
-            // new client2 is gone
-            final SettingsException e2 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure2"));
-            assertThat(e2.getMessage(), is("Unable to find client with name [azure2]"));
-            // client 3 emerged
-            final BlobServiceClient client23 = azureStorageService.client("azure3").v1();
-            assertThat(client23.getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
+                final BlobServiceClient client12 = azureStorageService.client("azure2").v1();
+                assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
+                // client 3 is missing
+                final SettingsException e1 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure3"));
+                assertThat(e1.getMessage(), is("Unable to find client with name [azure3]"));
+                // update client settings
+                plugin.reload(settings2);
+                // old client 1 not changed
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
+                // new client 1 is changed
+                final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
+                assertThat(client21.getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net"));
+                // old client 2 not changed
+                assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
+                // new client2 is gone
+                final SettingsException e2 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure2"));
+                assertThat(e2.getMessage(), is("Unable to find client with name [azure2]"));
+                // client 3 emerged
+                final BlobServiceClient client23 = azureStorageService.client("azure3").v1();
+                assertThat(client23.getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net"));
+            }
         }
     }
 
@@ -160,17 +255,18 @@ public void testReinitClientEmptySettings() throws IOException {
         secureSettings.setString("azure.client.azure1.key", encodeKey("mykey11"));
         final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            // reinit with empty settings
-            final SettingsException e = expectThrows(SettingsException.class, () -> plugin.reload(Settings.EMPTY));
-            assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration."));
-            // existing client untouched
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            // new client also untouched
-            final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
-            assertThat(client21.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+                // reinit with empty settings
+                final SettingsException e = expectThrows(SettingsException.class, () -> plugin.reload(Settings.EMPTY));
+                assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration."));
+                // existing client untouched
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+                // new client also untouched
+                final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
+                assertThat(client21.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            }
         }
     }
 
@@ -189,138 +285,150 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
         final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
-            assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
-            final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
-            assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
-            // existing client untouched
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+                final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
+                assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
+                final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
+                assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
+                // existing client untouched
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            }
         }
     }
 
-    public void testGetSelectedClientNonExisting() {
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
-        final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4"));
-        assertThat(e.getMessage(), is("Unable to find client with name [azure4]"));
+    public void testGetSelectedClientNonExisting() throws IOException {
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) {
+            final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4"));
+            assertThat(e.getMessage(), is("Unable to find client with name [azure4]"));
+        }
     }
 
-    public void testGetSelectedClientDefaultTimeout() {
+    public void testGetSelectedClientDefaultTimeout() throws IOException {
         final Settings timeoutSettings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.timeout", "30s")
             .build();
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings);
-        assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
-        assertThat(azureStorageService.getBlobRequestTimeout("azure3"), is(Duration.ofSeconds(30)));
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings)) {
+            assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
+            assertThat(azureStorageService.getBlobRequestTimeout("azure3"), is(Duration.ofSeconds(30)));
+        }
     }
 
-    public void testClientDefaultConnectTimeout() {
+    public void testClientDefaultConnectTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.connect.timeout", "25s")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getConnectTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getConnectTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueSeconds(25)));
-        assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), equalTo(TimeValue.timeValueSeconds(10)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueSeconds(25)));
+            assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), equalTo(TimeValue.timeValueSeconds(10)));
+        }
     }
 
-    public void testClientDefaultWriteTimeout() {
+    public void testClientDefaultWriteTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.write.timeout", "85s")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getWriteTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getWriteTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueSeconds(85)));
-        assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueSeconds(85)));
+            assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+        }
     }
 
-    public void testClientDefaultReadTimeout() {
+    public void testClientDefaultReadTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.read.timeout", "120s")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getReadTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getReadTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueSeconds(120)));
-        assertThat(mock.storageSettings.get("azure2").getReadTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getReadTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueSeconds(120)));
+            assertThat(mock.storageSettings.get("azure2").getReadTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getReadTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+        }
     }
 
-    public void testClientDefaultResponseTimeout() {
+    public void testClientDefaultResponseTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.response.timeout", "1ms")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getResponseTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getResponseTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueMillis(1)));
-        assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueMillis(1)));
+            assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+        }
     }
 
-    public void testGetSelectedClientNoTimeout() {
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
-        assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
+    public void testGetSelectedClientNoTimeout() throws IOException {
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) {
+            assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
+        }
     }
 
-    public void testGetSelectedClientBackoffPolicy() {
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
-        final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
-        assertThat(requestRetryOptions(client1), is(notNullValue()));
+    public void testGetSelectedClientBackoffPolicy() throws IOException {
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) {
+            final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+            assertThat(requestRetryOptions(client1), is(notNullValue()));
+        }
     }
 
-    public void testGetSelectedClientBackoffPolicyNbRetries() {
+    public void testGetSelectedClientBackoffPolicyNbRetries() throws IOException {
         final Settings timeoutSettings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.max_retries", 7)
             .build();
 
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings);
-        final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
-        assertThat(requestRetryOptions(client1), is(notNullValue()));
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings)) {
+            final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+            assertThat(requestRetryOptions(client1), is(notNullValue()));
+        }
     }
 
-    public void testNoProxy() {
+    public void testNoProxy() throws IOException {
         final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings()).build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        assertEquals(mock.storageSettings.get("azure1").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
-        assertEquals(mock.storageSettings.get("azure2").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
-        assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure1").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+            assertEquals(mock.storageSettings.get("azure2").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+            assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+        }
     }
 
-    public void testProxyHttp() throws UnknownHostException {
+    public void testProxyHttp() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.proxy.host", "127.0.0.1")
             .put("azure.client.azure1.proxy.port", 8080)
             .put("azure.client.azure1.proxy.type", "http")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
-
-        assertThat(azure1Proxy, notNullValue());
-        assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
-        assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
+
+            assertThat(azure1Proxy, notNullValue());
+            assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
+            assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        }
     }
 
-    public void testMultipleProxies() throws UnknownHostException {
+    public void testMultipleProxies() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.proxy.host", "127.0.0.1")
@@ -330,21 +438,22 @@ public void testMultipleProxies() throws UnknownHostException {
             .put("azure.client.azure2.proxy.port", 8081)
             .put("azure.client.azure2.proxy.type", "http")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
-        assertThat(azure1Proxy, notNullValue());
-        assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
-        assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
-        final ProxySettings azure2Proxy = mock.storageSettings.get("azure2").getProxySettings();
-        assertThat(azure2Proxy, notNullValue());
-        assertThat(azure2Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
-        assertThat(azure2Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081)));
-        assertTrue(Strings.isNullOrEmpty(azure2Proxy.getUsername()));
-        assertTrue(Strings.isNullOrEmpty(azure2Proxy.getPassword()));
-        assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
-    }
-
-    public void testProxySocks() throws UnknownHostException {
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
+            assertThat(azure1Proxy, notNullValue());
+            assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
+            assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
+            final ProxySettings azure2Proxy = mock.storageSettings.get("azure2").getProxySettings();
+            assertThat(azure2Proxy, notNullValue());
+            assertThat(azure2Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
+            assertThat(azure2Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081)));
+            assertTrue(Strings.isNullOrEmpty(azure2Proxy.getUsername()));
+            assertTrue(Strings.isNullOrEmpty(azure2Proxy.getPassword()));
+            assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+        }
+    }
+
+    public void testProxySocks() throws IOException {
         final MockSecureSettings secureSettings = buildSecureSettings();
         secureSettings.setString("azure.client.azure1.proxy.username", "user");
         secureSettings.setString("azure.client.azure1.proxy.password", "pwd");
@@ -354,15 +463,16 @@ public void testProxySocks() throws UnknownHostException {
             .put("azure.client.azure1.proxy.type", "socks5")
             .setSecureSettings(secureSettings)
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
-        assertThat(azure1Proxy, notNullValue());
-        assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.SOCKS5));
-        assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
-        assertEquals("user", azure1Proxy.getUsername());
-        assertEquals("pwd", azure1Proxy.getPassword());
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
+            assertThat(azure1Proxy, notNullValue());
+            assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.SOCKS5));
+            assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
+            assertEquals("user", azure1Proxy.getUsername());
+            assertEquals("pwd", azure1Proxy.getPassword());
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        }
     }
 
     public void testProxyNoHost() {
@@ -420,6 +530,199 @@ public void testBlobNameFromUri() throws URISyntaxException {
         assertThat(name, is("path/to/myfile"));
     }
 
+    public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+        secureSettings.setString("azure.client.azure2.account", "myaccount");
+
+        // Enabled Managed Identity in the settings using lower case and mixed case
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", "managed_identity")
+            .put("azure.client.azure2.token_credential_type", "managed_IDENTITY")
+            .build();
+
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed_identity");
+            assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "managed_IDENTITY");
+        }
+    }
+
+    public void testSettingTokenCredentialForAuthenticationWithAlternativeEnumValue() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Enabled Managed Identity in the settings using lower case
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", "managed")
+            .build();
+
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed");
+        }
+    }
+
+    public void testSettingUnsupportedTokenCredentialForAuthentication() {
+        final String unsupported_token_credential_type = "TOKEN_CREDENTIAL_TYPE_THAT_DOES_NOT_EXIST";
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Enable the unsupported token credential type
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", unsupported_token_credential_type)
+            .build();
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> storageServiceWithSettingsValidation(settings)
+        );
+        assertEquals(
+            "The token credential type '"
+                + unsupported_token_credential_type
+                + "' is unsupported, please use one of the following values: "
+                + String.join(", ", TokenCredentialType.getTokenCredentialTypes()),
+            e.getMessage()
+        );
+    }
+
+    public void testTokenCredentialAuthenticationOverridesOtherFormOfAuthentications() throws IOException {
+        final String token_credential_type = TokenCredentialType.MANAGED_IDENTITY.name();
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure1 with account key
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure1.key", encodeKey("mykey"));
+
+        // Azure 2 with sas token
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.sas_token", encodeKey("mysastoken"));
+
+        // Azure 3 with account key and sas token
+        secureSettings.setString("azure.client.azure3.account", "myaccount3");
+        secureSettings.setString("azure.client.azure3.key", encodeKey("mykey"));
+        secureSettings.setString("azure.client.azure3.sas_token", encodeKey("mysastoken"));
+
+        // Azure 4 without sas token and account key
+        secureSettings.setString("azure.client.azure4.account", "myaccount4");
+
+        // Enable Managed Identity in all azure clients
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure1.token_credential_type", token_credential_type)
+            .put("azure.client.azure2.token_credential_type", token_credential_type)
+            .put("azure.client.azure3.token_credential_type", token_credential_type)
+            .put("azure.client.azure4.token_credential_type", token_credential_type)
+            .build();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            // Expect token credential authentication is selected over account key or sas token.
+            assertEquals(token_credential_type, mock.storageSettings.get("azure1").getTokenCredentialType());
+            assertEquals(token_credential_type, mock.storageSettings.get("azure2").getTokenCredentialType());
+            assertEquals(token_credential_type, mock.storageSettings.get("azure3").getTokenCredentialType());
+            assertEquals(token_credential_type, mock.storageSettings.get("azure4").getTokenCredentialType());
+        }
+    }
+
+    public void testTokenCredentialWhenAccountIsNotProvided() {
+        // Setting with an account specified
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+
+        // Enabled Managed Identity in the settings
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+        final Exception e = expectThrows(Exception.class, () -> storageServiceWithSettingsValidation(settings));
+
+        // Expect failure due to missing account name
+        assertEquals(
+            "missing required setting [azure.client.azure.account] for setting [azure.client.azure.token_credential_type]",
+            e.getMessage()
+        );
+    }
+
+    public void testAuthenticationMethodNotProvided() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key and sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Disabled Managed Identity in the settings by default
+        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
+
+        // Expect fall back to authentication via sas token or account key when token credential is not specified.
+        assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage());
+    }
+
+    public void testSettingTokenCredentialTypeToBeEmpty() {
+        // Azure clients without account key and sas token.
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount");
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure1.token_credential_type", "")
+            .build();
+        // Expect fall back to authentication via sas token or account key when token credential is not specified.
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
+        assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage());
+
+        // Azure clients without account key and sas token.
+        final MockSecureSettings secureSettings2 = new MockSecureSettings();
+        secureSettings2.setString("azure.client.azure2.account", "myaccount");
+        final Settings settings2 = Settings.builder()
+            .setSecureSettings(secureSettings2)
+            .put("azure.client.azure2.token_credential_type", "x")
+            .build();
+        // Expect failing token credential type checks
+        final IllegalArgumentException e2 = expectThrows(
+            IllegalArgumentException.class,
+            () -> storageServiceWithSettingsValidation(settings2)
+        );
+        assertEquals(
+            "The token credential type 'x' is unsupported, please use one of the following values: "
+                + String.join(", ", TokenCredentialType.getTokenCredentialTypes()),
+            e2.getMessage()
+        );
+    }
+
+    public void testManagedIdentityIsEnabled() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Enabled Managed Identity in the settings.
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), TokenCredentialType.MANAGED_IDENTITY.name());
+        }
+    }
+
+    public void testNonTokenCredentialAuthenticationEnabled() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client account key.
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure1.sas_token", encodeKey("mysastoken"));
+
+        // Azure client with sas token
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.key", encodeKey("mykey"));
+
+        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            // Expect token credential is not enabled
+            assertEquals(mock.storageSettings.get("azure1").getTokenCredentialType(), "");
+            assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "");
+        }
+    }
+
     private static MockSecureSettings buildSecureSettings() {
         final MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("azure.client.azure1.account", "myaccount1");
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index e1226345ef961..55e8db0d9c6a3 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -85,6 +85,11 @@ grant codeBase "${codebase.zstd-jni}" {
   permission java.lang.RuntimePermission "loadLibrary.*";
 };
 
+// repository-azure plugin and server side streaming
+grant codeBase "${codebase.reactor-core}" {
+  permission java.net.SocketPermission "*", "connect,resolve";
+};
+
 //// Everything else:
 
 grant {

From c8449e50c755a9b729d0609b76ac58b4352905ac Mon Sep 17 00:00:00 2001
From: "opensearch-trigger-bot[bot]"
 <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com>
Date: Wed, 15 May 2024 17:28:09 -0400
Subject: [PATCH 35/52] [AUTO] [main] Add bwc version 2.14.1. (#13673)

* Add bwc version 2.14.1

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update Version.java

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: opensearch-ci-bot <83309141+opensearch-ci-bot@users.noreply.github.com>
Co-authored-by: Andriy Redko <andriy.redko@aiven.io>
---
 .ci/bwcVersions                                     | 1 +
 README.md                                           | 2 +-
 libs/core/src/main/java/org/opensearch/Version.java | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index e490c6ae6b4ea..398c3d52595f1 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -32,4 +32,5 @@ BWC_VERSION:
   - "2.13.0"
   - "2.13.1"
   - "2.14.0"
+  - "2.14.1"
   - "2.15.0"
diff --git a/README.md b/README.md
index aff18018960d5..17af2911b9221 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.14.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.0")
+[![2.14.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.1")
 [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index 823eb110eaa10..3f680b4ab8e05 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -103,6 +103,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_13_1 = new Version(2130199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_3_0_0;

From febe8c71a8d5a10803934a98938d4a07b57daa7f Mon Sep 17 00:00:00 2001
From: Craig Perkins <cwperx@amazon.com>
Date: Wed, 15 May 2024 17:28:21 -0400
Subject: [PATCH 36/52] Bump mockito from 5.11.0 to 5.12.0 (#13665)

* Bump mockito from 5.11.0 to 5.12.0

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add to CHANGELOG

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
---
 CHANGELOG.md                | 1 +
 buildSrc/version.properties | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65c5fa1223cc6..1f868dd76039b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
 - Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
+- Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d0aaea546803b..1e097d586aded 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -57,7 +57,7 @@ bouncycastle=1.78
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-mockito           = 5.11.0
+mockito           = 5.12.0
 objenesis         = 3.2
 bytebuddy         = 1.14.9
 

From a9298c95718b5d7a9cbfc75a0f1874737f842c2d Mon Sep 17 00:00:00 2001
From: Prudhvi Godithi <pgodithi@amazon.com>
Date: Wed, 15 May 2024 16:14:22 -0700
Subject: [PATCH 37/52] Update Pull Request Checks to match the (#13688)

---
 .github/workflows/dependabot_pr.yml       | 2 +-
 .github/workflows/pull-request-checks.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml
index e6feb3b852ad0..cfd3ac81c5bd3 100644
--- a/.github/workflows/dependabot_pr.yml
+++ b/.github/workflows/dependabot_pr.yml
@@ -7,7 +7,7 @@ jobs:
     permissions:
       pull-requests: write
       contents: write
-    if: ${{ github.actor == 'dependabot[bot]' }}
+    if: ${{ github.event.pull_request.user.login == 'dependabot[bot]' }}
     steps:
       - name: GitHub App token
         id: github_app_token
diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
index a62ea9cfa179b..eec363572478c 100644
--- a/.github/workflows/pull-request-checks.yml
+++ b/.github/workflows/pull-request-checks.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: peternied/check-pull-request-description-checklist@v1.1
-        if: github.actor != 'dependabot[bot]'
+        if: github.event.pull_request.user.login != 'dependabot[bot]'
         with:
           checklist-items: |
             New functionality includes testing.

From da3ab920fcd0f6a8f5dbfad5c87740261f355f14 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <31575408+shiv0408@users.noreply.github.com>
Date: Thu, 16 May 2024 12:39:56 +0530
Subject: [PATCH 38/52] Split the remote global metadata file to metadata
 attribute files (#12190)

* Split the cluster state remote global metadata file to metadata attribute files

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
---
 .../opensearch/upgrades/ClusterStateIT.java   |  37 +
 .../remote/RemoteClusterStateServiceIT.java   |  47 ++
 .../RemoteStoreClusterStateRestoreIT.java     |  79 +-
 .../opensearch/cluster/metadata/Metadata.java |  75 +-
 .../metadata/RepositoriesMetadata.java        |   4 +
 .../cluster/metadata/TemplatesMetadata.java   | 150 ++++
 .../remote/ClusterMetadataManifest.java       | 356 ++++++++-
 .../remote/RemoteClusterStateService.java     | 692 ++++++++++++-----
 .../coordination/CoordinationStateTests.java  |  28 +-
 .../remote/ClusterMetadataManifestTests.java  | 116 ++-
 .../RemoteClusterStateServiceTests.java       | 719 ++++++++++++++----
 11 files changed, 1883 insertions(+), 420 deletions(-)
 create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java

diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java
new file mode 100644
index 0000000000000..2d606d27a34e0
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.upgrades;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.util.Map;
+
+public class ClusterStateIT extends AbstractRollingTestCase{
+    public void testTemplateMetadataUpgrades() throws Exception {
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            String templateName = "my_template";
+            Request putIndexTemplate = new Request("PUT", "_template/" + templateName);
+            putIndexTemplate.setJsonEntity("{\"index_patterns\": [\"pattern-1\", \"log-*\"]}");
+            client().performRequest(putIndexTemplate);
+            verifyTemplateMetadataInClusterState();
+        } else {
+            verifyTemplateMetadataInClusterState();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void verifyTemplateMetadataInClusterState() throws Exception {
+        Request request = new Request("GET", "_cluster/state/metadata");
+        Response response = client().performRequest(request);
+        assertOK(response);
+        Map<String, Object> metadata = (Map<String, Object>) entityAsMap(response).get("metadata");
+        assertNotNull(metadata.get("templates"));
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
index dfde1b958882c..61b34af5be3ba 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
@@ -19,13 +19,21 @@
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase {
@@ -181,6 +189,45 @@ public void testRemoteStateStatsFromAllNodes() {
         }
     }
 
+    public void testRemoteClusterStateMetadataSplit() throws IOException {
+        initialTestSetup(1, 0, 1, 1);
+
+        RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance(
+            RemoteClusterStateService.class
+        );
+        RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class);
+        BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME);
+        BlobPath globalMetadataPath = repository.basePath()
+            .add(
+                Base64.getUrlEncoder()
+                    .withoutPadding()
+                    .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8))
+            )
+            .add("cluster-state")
+            .add(getClusterState().metadata().clusterUUID())
+            .add("global-metadata");
+
+        Map<String, Integer> metadataFiles = repository.blobStore()
+            .blobContainer(globalMetadataPath)
+            .listBlobs()
+            .keySet()
+            .stream()
+            .map(fileName -> {
+                logger.info(fileName);
+                return fileName.split(DELIMITER)[0];
+            })
+            .collect(Collectors.toMap(Function.identity(), key -> 1, Integer::sum));
+
+        assertTrue(metadataFiles.containsKey(COORDINATION_METADATA));
+        assertEquals(1, (int) metadataFiles.get(COORDINATION_METADATA));
+        assertTrue(metadataFiles.containsKey(SETTING_METADATA));
+        assertEquals(1, (int) metadataFiles.get(SETTING_METADATA));
+        assertTrue(metadataFiles.containsKey(TEMPLATES_METADATA));
+        assertEquals(1, (int) metadataFiles.get(TEMPLATES_METADATA));
+        assertTrue(metadataFiles.keySet().stream().anyMatch(key -> key.startsWith(CUSTOM_METADATA)));
+        assertFalse(metadataFiles.containsKey(METADATA_FILE_PREFIX));
+    }
+
     private void validateNodesStatsResponse(NodesStatsResponse nodesStatsResponse) {
         // _nodes/stats/discovery must never fail due to any exception
         assertFalse(nodesStatsResponse.toString().contains("exception"));
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index 3f90732f1f13d..8c8209b80bfd8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -8,16 +8,30 @@
 
 package org.opensearch.remotestore;
 
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.datastream.DataStreamRolloverIT;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction;
+import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.block.ClusterBlockException;
+import org.opensearch.cluster.metadata.ComponentTemplate;
+import org.opensearch.cluster.metadata.ComponentTemplateMetadata;
+import org.opensearch.cluster.metadata.ComposableIndexTemplate;
+import org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexTemplateMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
+import org.opensearch.cluster.metadata.Template;
+import org.opensearch.common.action.ActionFuture;
+import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
@@ -29,11 +43,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING;
 import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING;
@@ -46,6 +62,11 @@
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT {
+    static final String TEMPLATE_NAME = "remote-store-test-template";
+    static final String COMPONENT_TEMPLATE_NAME = "remote-component-template1";
+    static final String COMPOSABLE_TEMPLATE_NAME = "remote-composable-template1";
+    static final Setting<String> MOCK_SETTING = Setting.simpleString("mock-setting");
+    static final String[] EXCLUDED_NODES = { "ex-1", "ex-2" };
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -87,6 +108,45 @@ public void testFullClusterRestore() throws Exception {
         Map<String, Long> indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1);
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
         long prevClusterStateVersion = clusterService().state().version();
+        // Step - 1.1 Add some cluster state elements
+        ActionFuture<AcknowledgedResponse> response = client().admin()
+            .indices()
+            .preparePutTemplate(TEMPLATE_NAME)
+            .addAlias(new Alias(INDEX_NAME))
+            .setPatterns(Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList()))
+            .execute();
+        assertTrue(response.get().isAcknowledged());
+        ActionFuture<ClusterUpdateSettingsResponse> clusterUpdateSettingsResponse = client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), false).build())
+            .execute();
+        assertTrue(clusterUpdateSettingsResponse.get().isAcknowledged());
+        // update coordination metadata
+        client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(EXCLUDED_NODES));
+        // Add a custom metadata as component index template
+        ActionFuture<AcknowledgedResponse> componentTemplateResponse = client().execute(
+            PutComponentTemplateAction.INSTANCE,
+            new PutComponentTemplateAction.Request(COMPONENT_TEMPLATE_NAME).componentTemplate(
+                new ComponentTemplate(new Template(Settings.EMPTY, null, Collections.emptyMap()), 1L, Collections.emptyMap())
+            )
+        );
+        assertTrue(componentTemplateResponse.get().isAcknowledged());
+        ActionFuture<AcknowledgedResponse> composableTemplateResponse = client().execute(
+            PutComposableIndexTemplateAction.INSTANCE,
+            new PutComposableIndexTemplateAction.Request(COMPOSABLE_TEMPLATE_NAME).indexTemplate(
+                new ComposableIndexTemplate(
+                    Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList()),
+                    new Template(Settings.EMPTY, null, Collections.emptyMap()),
+                    Collections.singletonList(COMPONENT_TEMPLATE_NAME),
+                    1L,
+                    1L,
+                    Collections.emptyMap(),
+                    null
+                )
+            )
+        );
+        assertTrue(composableTemplateResponse.get().isAcknowledged());
 
         // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata
         resetCluster(dataNodeCount, clusterManagerNodeCount);
@@ -104,7 +164,24 @@ public void testFullClusterRestore() throws Exception {
         );
         validateMetadata(List.of(INDEX_NAME));
         verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
-
+        clusterService().state()
+            .metadata()
+            .coordinationMetadata()
+            .getVotingConfigExclusions()
+            .stream()
+            .forEach(config -> assertTrue(Arrays.stream(EXCLUDED_NODES).anyMatch(node -> node.equals(config.getNodeId()))));
+        assertFalse(clusterService().state().metadata().templates().isEmpty());
+        assertTrue(clusterService().state().metadata().templates().containsKey(TEMPLATE_NAME));
+        assertFalse(clusterService().state().metadata().settings().isEmpty());
+        assertFalse(clusterService().state().metadata().settings().getAsBoolean(SETTING_READ_ONLY_SETTING.getKey(), true));
+        assertNotNull(clusterService().state().metadata().custom("component_template"));
+        ComponentTemplateMetadata componentTemplateMetadata = clusterService().state().metadata().custom("component_template");
+        assertFalse(componentTemplateMetadata.componentTemplates().isEmpty());
+        assertTrue(componentTemplateMetadata.componentTemplates().containsKey(COMPONENT_TEMPLATE_NAME));
+        assertNotNull(clusterService().state().metadata().custom("index_template"));
+        ComposableIndexTemplateMetadata composableIndexTemplate = clusterService().state().metadata().custom("index_template");
+        assertFalse(composableIndexTemplate.indexTemplates().isEmpty());
+        assertTrue(composableIndexTemplate.indexTemplates().containsKey(COMPOSABLE_TEMPLATE_NAME));
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
index 59dc86ea28ed6..d016501dd0910 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
@@ -175,6 +175,16 @@ public enum XContentContext {
     public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, ClusterState.FeatureAware {
 
         EnumSet<XContentContext> context();
+
+        static Custom fromXContent(XContentParser parser, String name) throws IOException {
+            // handling any Exception is caller's responsibility
+            return parser.namedObject(Custom.class, name, null);
+        }
+
+        static Custom fromXContent(XContentParser parser) throws IOException {
+            String currentFieldName = parser.currentName();
+            return fromXContent(parser, currentFieldName);
+        }
     }
 
     public static final Setting<Integer> DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting(
@@ -260,7 +270,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
     private final Settings settings;
     private final DiffableStringMap hashesOfConsistentSettings;
     private final Map<String, IndexMetadata> indices;
-    private final Map<String, IndexTemplateMetadata> templates;
+    private final TemplatesMetadata templates;
     private final Map<String, Custom> customs;
 
     private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
@@ -304,7 +314,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
         this.hashesOfConsistentSettings = hashesOfConsistentSettings;
         this.indices = Collections.unmodifiableMap(indices);
         this.customs = Collections.unmodifiableMap(customs);
-        this.templates = Collections.unmodifiableMap(templates);
+        this.templates = new TemplatesMetadata(templates);
         int totalNumberOfShards = 0;
         int totalOpenIndexShards = 0;
         for (IndexMetadata cursor : indices.values()) {
@@ -806,13 +816,17 @@ public Map<String, IndexMetadata> getIndices() {
     }
 
     public Map<String, IndexTemplateMetadata> templates() {
-        return this.templates;
+        return this.templates.getTemplates();
     }
 
     public Map<String, IndexTemplateMetadata> getTemplates() {
         return templates();
     }
 
+    public TemplatesMetadata templatesMetadata() {
+        return this.templates;
+    }
+
     public Map<String, ComponentTemplate> componentTemplates() {
         return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
             .map(ComponentTemplateMetadata::componentTemplates)
@@ -927,7 +941,7 @@ public Iterator<IndexMetadata> iterator() {
     }
 
     public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
+        if (!isCoordinationMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
@@ -946,13 +960,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
      * Compares Metadata entities persisted in Remote Store.
      */
     public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
+        if (!isSettingsMetadataEqual(metadata1, metadata2)) {
             return false;
         }
-        if (!metadata1.templates.equals(metadata2.templates())) {
+        if (!isTemplatesMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         // Check if any persistent metadata needs to be saved
+        return isCustomMetadataEqual(metadata1, metadata2);
+    }
+
+    public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata);
+    }
+
+    public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.persistentSettings.equals(metadata2.persistentSettings);
+    }
+
+    public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.templates.equals(metadata2.templates);
+    }
+
+    public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
         int customCount1 = 0;
         for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
             if (cursor.getValue().context().contains(XContentContext.GATEWAY)) {
@@ -966,8 +996,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
                 customCount2++;
             }
         }
-        if (customCount1 != customCount2) return false;
-        return true;
+        return customCount1 == customCount2;
     }
 
     @Override
@@ -1016,7 +1045,11 @@ private static class MetadataDiff implements Diff<Metadata> {
             persistentSettings = after.persistentSettings;
             hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
-            templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
+            templates = DiffableUtils.diff(
+                before.templates.getTemplates(),
+                after.templates.getTemplates(),
+                DiffableUtils.getStringKeySerializer()
+            );
             customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
@@ -1063,7 +1096,7 @@ public Metadata apply(Metadata part) {
             builder.persistentSettings(persistentSettings);
             builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings));
             builder.indices(indices.apply(part.indices));
-            builder.templates(templates.apply(part.templates));
+            builder.templates(templates.apply(part.templates.getTemplates()));
             builder.customs(customs.apply(part.customs));
             return builder.build();
         }
@@ -1107,10 +1140,7 @@ public void writeTo(StreamOutput out) throws IOException {
         for (IndexMetadata indexMetadata : this) {
             indexMetadata.writeTo(out);
         }
-        out.writeVInt(templates.size());
-        for (final IndexTemplateMetadata cursor : templates.values()) {
-            cursor.writeTo(out);
-        }
+        templates.writeTo(out);
         // filter out custom states not supported by the other node
         int numberOfCustoms = 0;
         for (final Custom cursor : customs.values()) {
@@ -1174,7 +1204,7 @@ public Builder(Metadata metadata) {
             this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings;
             this.version = metadata.version;
             this.indices = new HashMap<>(metadata.indices);
-            this.templates = new HashMap<>(metadata.templates);
+            this.templates = new HashMap<>(metadata.templates.getTemplates());
             this.customs = new HashMap<>(metadata.customs);
             this.previousMetadata = metadata;
         }
@@ -1253,6 +1283,11 @@ public Builder templates(Map<String, IndexTemplateMetadata> templates) {
             return this;
         }
 
+        public Builder templates(TemplatesMetadata templatesMetadata) {
+            this.templates.putAll(templatesMetadata.getTemplates());
+            return this;
+        }
+
         public Builder put(String name, ComponentTemplate componentTemplate) {
             Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
             Map<String, ComponentTemplate> existingTemplates = Optional.ofNullable(
@@ -1773,9 +1808,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
             }
 
             builder.startObject("templates");
-            for (final IndexTemplateMetadata cursor : metadata.templates().values()) {
-                IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
-            }
+            metadata.templatesMetadata().toXContent(builder, params);
             builder.endObject();
 
             if (context == XContentContext.API) {
@@ -1838,12 +1871,10 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
                     } else if ("hashes_of_consistent_settings".equals(currentFieldName)) {
                         builder.hashesOfConsistentSettings(parser.mapStrings());
                     } else if ("templates".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
-                        }
+                        builder.templates(TemplatesMetadata.fromXContent(parser));
                     } else {
                         try {
-                            Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
+                            Custom custom = Custom.fromXContent(parser, currentFieldName);
                             builder.putCustom(custom.getWriteableName(), custom);
                         } catch (NamedObjectNotFoundException ex) {
                             logger.warn("Skipping unknown custom object with type {}", currentFieldName);
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java
index e3689d046193c..9b52bdd1b16c5 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java
@@ -202,6 +202,10 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
         XContentParser.Token token;
         List<RepositoryMetadata> repository = new ArrayList<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                // move to next token if parsing the whole object
+                token = parser.nextToken();
+            }
             if (token == XContentParser.Token.FIELD_NAME) {
                 String name = parser.currentName();
                 if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java
new file mode 100644
index 0000000000000..6ecc471c5e0ae
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java
@@ -0,0 +1,150 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.cluster.AbstractDiffable;
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Metadata for legacy templates
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "2.15.0")
+public class TemplatesMetadata extends AbstractDiffable<TemplatesMetadata> implements ToXContentFragment {
+    public static TemplatesMetadata EMPTY_METADATA = builder().build();
+    private final Map<String, IndexTemplateMetadata> templates;
+
+    public TemplatesMetadata() {
+        this(Collections.emptyMap());
+    }
+
+    public TemplatesMetadata(Map<String, IndexTemplateMetadata> templates) {
+        this.templates = Collections.unmodifiableMap(templates);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public Map<String, IndexTemplateMetadata> getTemplates() {
+        return this.templates;
+    }
+
+    public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
+        return Builder.fromXContent(parser);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        Builder.toXContent(this, builder, params);
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(templates.size());
+        for (final IndexTemplateMetadata cursor : templates.values()) {
+            cursor.writeTo(out);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        TemplatesMetadata that = (TemplatesMetadata) o;
+
+        return Objects.equals(templates, that.templates);
+    }
+
+    @Override
+    public int hashCode() {
+        return templates != null ? templates.hashCode() : 0;
+    }
+
+    /**
+     * Builder for the templates metadata
+     *
+     * @opensearch.api
+     */
+    @PublicApi(since = "2.15.0")
+    public static class Builder {
+        private final Map<String, IndexTemplateMetadata> templates;
+
+        public Builder() {
+            this.templates = new HashMap<String, IndexTemplateMetadata>();
+        }
+
+        public Builder(Map<String, IndexTemplateMetadata> templates) {
+            this.templates = templates;
+        }
+
+        public Builder put(IndexTemplateMetadata.Builder templateBuilder) {
+            return put(templateBuilder.build());
+        }
+
+        public Builder put(IndexTemplateMetadata template) {
+            templates.put(template.name(), template);
+            return this;
+        }
+
+        public Builder removeTemplate(String templateName) {
+            templates.remove(templateName);
+            return this;
+        }
+
+        public Builder templates(Map<String, IndexTemplateMetadata> templates) {
+            this.templates.putAll(templates);
+            return this;
+        }
+
+        public TemplatesMetadata build() {
+            return new TemplatesMetadata(templates);
+        }
+
+        public static void toXContent(TemplatesMetadata templatesMetadata, XContentBuilder builder, Params params) throws IOException {
+            for (IndexTemplateMetadata cursor : templatesMetadata.getTemplates().values()) {
+                IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
+            }
+        }
+
+        public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
+            Builder builder = new Builder();
+
+            XContentParser.Token token = parser.currentToken();
+            String currentFieldName = parser.currentName();
+            if (currentFieldName == null) {
+                token = parser.nextToken();
+                if (token == XContentParser.Token.START_OBJECT) {
+                    // move to the field name
+                    token = parser.nextToken();
+                }
+                currentFieldName = parser.currentName();
+            }
+            if (currentFieldName != null) {
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
+                }
+            }
+            return builder.build();
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
index 4725f40076ce2..bf02c73ca560b 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
@@ -16,6 +16,7 @@
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ObjectParser;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
@@ -23,8 +24,12 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Manifest file which contains the details of the uploaded entity metadata
@@ -35,6 +40,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
 
     public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest.
     public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file.
+    public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file.
 
     private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term");
     private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version");
@@ -48,6 +54,37 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
     private static final ParseField INDICES_FIELD = new ParseField("indices");
     private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid");
     private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed");
+    private static final ParseField UPLOADED_COORDINATOR_METADATA = new ParseField("uploaded_coordinator_metadata");
+    private static final ParseField UPLOADED_SETTINGS_METADATA = new ParseField("uploaded_settings_metadata");
+    private static final ParseField UPLOADED_TEMPLATES_METADATA = new ParseField("uploaded_templates_metadata");
+    private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata");
+
+    private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) {
+        return ClusterMetadataManifest.builder()
+            .clusterTerm(term(fields))
+            .stateVersion(version(fields))
+            .clusterUUID(clusterUUID(fields))
+            .stateUUID(stateUUID(fields))
+            .opensearchVersion(opensearchVersion(fields))
+            .nodeId(nodeId(fields))
+            .committed(committed(fields))
+            .codecVersion(CODEC_V0)
+            .indices(indices(fields))
+            .previousClusterUUID(previousClusterUUID(fields))
+            .clusterUUIDCommitted(clusterUUIDCommitted(fields));
+    }
+
+    private static ClusterMetadataManifest.Builder manifestV1Builder(Object[] fields) {
+        return manifestV0Builder(fields).codecVersion(codecVersion(fields)).globalMetadataFileName(globalMetadataFileName(fields));
+    }
+
+    private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields) {
+        return manifestV0Builder(fields).codecVersion(codecVersion(fields))
+            .coordinationMetadata(coordinationMetadata(fields))
+            .settingMetadata(settingsMetadata(fields))
+            .templatesMetadata(templatesMetadata(fields))
+            .customMetadataMap(customMetadata(fields));
+    }
 
     private static long term(Object[] fields) {
         return (long) fields[0];
@@ -97,47 +134,44 @@ private static String globalMetadataFileName(Object[] fields) {
         return (String) fields[11];
     }
 
+    private static UploadedMetadataAttribute coordinationMetadata(Object[] fields) {
+        return (UploadedMetadataAttribute) fields[11];
+    }
+
+    private static UploadedMetadataAttribute settingsMetadata(Object[] fields) {
+        return (UploadedMetadataAttribute) fields[12];
+    }
+
+    private static UploadedMetadataAttribute templatesMetadata(Object[] fields) {
+        return (UploadedMetadataAttribute) fields[13];
+    }
+
+    private static Map<String, UploadedMetadataAttribute> customMetadata(Object[] fields) {
+        List<UploadedMetadataAttribute> customs = (List<UploadedMetadataAttribute>) fields[14];
+        return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()));
+    }
+
     private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER_V0 = new ConstructingObjectParser<>(
         "cluster_metadata_manifest",
-        fields -> new ClusterMetadataManifest(
-            term(fields),
-            version(fields),
-            clusterUUID(fields),
-            stateUUID(fields),
-            opensearchVersion(fields),
-            nodeId(fields),
-            committed(fields),
-            CODEC_V0,
-            null,
-            indices(fields),
-            previousClusterUUID(fields),
-            clusterUUIDCommitted(fields)
-        )
+        fields -> manifestV0Builder(fields).build()
     );
 
     private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER_V1 = new ConstructingObjectParser<>(
         "cluster_metadata_manifest",
-        fields -> new ClusterMetadataManifest(
-            term(fields),
-            version(fields),
-            clusterUUID(fields),
-            stateUUID(fields),
-            opensearchVersion(fields),
-            nodeId(fields),
-            committed(fields),
-            codecVersion(fields),
-            globalMetadataFileName(fields),
-            indices(fields),
-            previousClusterUUID(fields),
-            clusterUUIDCommitted(fields)
-        )
+        fields -> manifestV1Builder(fields).build()
     );
 
-    private static final ConstructingObjectParser<ClusterMetadataManifest, Void> CURRENT_PARSER = PARSER_V1;
+    private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER_V2 = new ConstructingObjectParser<>(
+        "cluster_metadata_manifest",
+        fields -> manifestV2Builder(fields).build()
+    );
+
+    private static final ConstructingObjectParser<ClusterMetadataManifest, Void> CURRENT_PARSER = PARSER_V2;
 
     static {
         declareParser(PARSER_V0, CODEC_V0);
         declareParser(PARSER_V1, CODEC_V1);
+        declareParser(PARSER_V2, CODEC_V2);
     }
 
     private static void declareParser(ConstructingObjectParser<ClusterMetadataManifest, Void> parser, long codec_version) {
@@ -156,14 +190,40 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
         parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
         parser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_COMMITTED);
 
-        if (codec_version >= CODEC_V1) {
+        if (codec_version == CODEC_V1) {
             parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD);
             parser.declareString(ConstructingObjectParser.constructorArg(), GLOBAL_METADATA_FIELD);
+        } else if (codec_version >= CODEC_V2) {
+            parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD);
+            parser.declareNamedObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_COORDINATOR_METADATA
+            );
+            parser.declareNamedObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_SETTINGS_METADATA
+            );
+            parser.declareNamedObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_TEMPLATES_METADATA
+            );
+            parser.declareNamedObjects(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_CUSTOM_METADATA
+            );
         }
     }
 
     private final int codecVersion;
     private final String globalMetadataFileName;
+    private final UploadedMetadataAttribute uploadedCoordinationMetadata;
+    private final UploadedMetadataAttribute uploadedSettingsMetadata;
+    private final UploadedMetadataAttribute uploadedTemplatesMetadata;
+    private final Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap;
     private final List<UploadedIndexMetadata> indices;
     private final long clusterTerm;
     private final long stateVersion;
@@ -223,6 +283,29 @@ public String getGlobalMetadataFileName() {
         return globalMetadataFileName;
     }
 
+    public UploadedMetadataAttribute getCoordinationMetadata() {
+        return uploadedCoordinationMetadata;
+    }
+
+    public UploadedMetadataAttribute getSettingsMetadata() {
+        return uploadedSettingsMetadata;
+    }
+
+    public UploadedMetadataAttribute getTemplatesMetadata() {
+        return uploadedTemplatesMetadata;
+    }
+
+    public Map<String, UploadedMetadataAttribute> getCustomMetadataMap() {
+        return uploadedCustomMetadataMap;
+    }
+
+    public boolean hasMetadataAttributesFiles() {
+        return uploadedCoordinationMetadata != null
+            || uploadedSettingsMetadata != null
+            || uploadedTemplatesMetadata != null
+            || !uploadedCustomMetadataMap.isEmpty();
+    }
+
     public ClusterMetadataManifest(
         long clusterTerm,
         long version,
@@ -235,7 +318,11 @@ public ClusterMetadataManifest(
         String globalMetadataFileName,
         List<UploadedIndexMetadata> indices,
         String previousClusterUUID,
-        boolean clusterUUIDCommitted
+        boolean clusterUUIDCommitted,
+        UploadedMetadataAttribute uploadedCoordinationMetadata,
+        UploadedMetadataAttribute uploadedSettingsMetadata,
+        UploadedMetadataAttribute uploadedTemplatesMetadata,
+        Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap
     ) {
         this.clusterTerm = clusterTerm;
         this.stateVersion = version;
@@ -249,6 +336,12 @@ public ClusterMetadataManifest(
         this.indices = Collections.unmodifiableList(indices);
         this.previousClusterUUID = previousClusterUUID;
         this.clusterUUIDCommitted = clusterUUIDCommitted;
+        this.uploadedCoordinationMetadata = uploadedCoordinationMetadata;
+        this.uploadedSettingsMetadata = uploadedSettingsMetadata;
+        this.uploadedTemplatesMetadata = uploadedTemplatesMetadata;
+        this.uploadedCustomMetadataMap = Collections.unmodifiableMap(
+            uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>()
+        );
     }
 
     public ClusterMetadataManifest(StreamInput in) throws IOException {
@@ -262,12 +355,29 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
         this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
         this.previousClusterUUID = in.readString();
         this.clusterUUIDCommitted = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            this.codecVersion = in.readInt();
+            this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in);
+            this.uploadedSettingsMetadata = new UploadedMetadataAttribute(in);
+            this.uploadedTemplatesMetadata = new UploadedMetadataAttribute(in);
+            this.uploadedCustomMetadataMap = Collections.unmodifiableMap(
+                in.readMap(StreamInput::readString, UploadedMetadataAttribute::new)
+            );
+            this.globalMetadataFileName = null;
+        } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             this.codecVersion = in.readInt();
             this.globalMetadataFileName = in.readString();
+            this.uploadedCoordinationMetadata = null;
+            this.uploadedSettingsMetadata = null;
+            this.uploadedTemplatesMetadata = null;
+            this.uploadedCustomMetadataMap = null;
         } else {
             this.codecVersion = CODEC_V0; // Default codec
             this.globalMetadataFileName = null;
+            this.uploadedCoordinationMetadata = null;
+            this.uploadedSettingsMetadata = null;
+            this.uploadedTemplatesMetadata = null;
+            this.uploadedCustomMetadataMap = null;
         }
     }
 
@@ -297,7 +407,29 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.endArray();
         builder.field(PREVIOUS_CLUSTER_UUID.getPreferredName(), getPreviousClusterUUID());
         builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted());
-        if (onOrAfterCodecVersion(CODEC_V1)) {
+        if (onOrAfterCodecVersion(CODEC_V2)) {
+            builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion());
+            if (getCoordinationMetadata() != null) {
+                builder.startObject(UPLOADED_COORDINATOR_METADATA.getPreferredName());
+                getCoordinationMetadata().toXContent(builder, params);
+                builder.endObject();
+            }
+            if (getSettingsMetadata() != null) {
+                builder.startObject(UPLOADED_SETTINGS_METADATA.getPreferredName());
+                getSettingsMetadata().toXContent(builder, params);
+                builder.endObject();
+            }
+            if (getTemplatesMetadata() != null) {
+                builder.startObject(UPLOADED_TEMPLATES_METADATA.getPreferredName());
+                getTemplatesMetadata().toXContent(builder, params);
+                builder.endObject();
+            }
+            builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName());
+            for (UploadedMetadataAttribute attribute : getCustomMetadataMap().values()) {
+                attribute.toXContent(builder, params);
+            }
+            builder.endObject();
+        } else if (onOrAfterCodecVersion(CODEC_V1)) {
             builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion());
             builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName());
         }
@@ -316,7 +448,13 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(indices);
         out.writeString(previousClusterUUID);
         out.writeBoolean(clusterUUIDCommitted);
-        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeInt(codecVersion);
+            uploadedCoordinationMetadata.writeTo(out);
+            uploadedSettingsMetadata.writeTo(out);
+            uploadedTemplatesMetadata.writeTo(out);
+            out.writeMap(uploadedCustomMetadataMap, StreamOutput::writeString, (o, v) -> v.writeTo(o));
+        } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeInt(codecVersion);
             out.writeString(globalMetadataFileName);
         }
@@ -376,6 +514,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro
         return PARSER_V0.parse(parser, null);
     }
 
+    public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException {
+        return PARSER_V1.parse(parser, null);
+    }
+
     public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException {
         return CURRENT_PARSER.parse(parser, null);
     }
@@ -388,6 +530,10 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws
     public static class Builder {
 
         private String globalMetadataFileName;
+        private UploadedMetadataAttribute coordinationMetadata;
+        private UploadedMetadataAttribute settingsMetadata;
+        private UploadedMetadataAttribute templatesMetadata;
+        private Map<String, UploadedMetadataAttribute> customMetadataMap;
         private int codecVersion;
         private List<UploadedIndexMetadata> indices;
         private long clusterTerm;
@@ -415,6 +561,31 @@ public Builder globalMetadataFileName(String globalMetadataFileName) {
             return this;
         }
 
+        public Builder coordinationMetadata(UploadedMetadataAttribute coordinationMetadata) {
+            this.coordinationMetadata = coordinationMetadata;
+            return this;
+        }
+
+        public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) {
+            this.settingsMetadata = settingsMetadata;
+            return this;
+        }
+
+        public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) {
+            this.templatesMetadata = templatesMetadata;
+            return this;
+        }
+
+        public Builder customMetadataMap(Map<String, UploadedMetadataAttribute> customMetadataMap) {
+            this.customMetadataMap = customMetadataMap;
+            return this;
+        }
+
+        public Builder put(String custom, UploadedMetadataAttribute customMetadata) {
+            this.customMetadataMap.put(custom, customMetadata);
+            return this;
+        }
+
         public Builder clusterTerm(long clusterTerm) {
             this.clusterTerm = clusterTerm;
             return this;
@@ -466,6 +637,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) {
 
         public Builder() {
             indices = new ArrayList<>();
+            customMetadataMap = new HashMap<>();
         }
 
         public Builder(ClusterMetadataManifest manifest) {
@@ -477,6 +649,10 @@ public Builder(ClusterMetadataManifest manifest) {
             this.nodeId = manifest.nodeId;
             this.committed = manifest.committed;
             this.globalMetadataFileName = manifest.globalMetadataFileName;
+            this.coordinationMetadata = manifest.uploadedCoordinationMetadata;
+            this.settingsMetadata = manifest.uploadedSettingsMetadata;
+            this.templatesMetadata = manifest.uploadedTemplatesMetadata;
+            this.customMetadataMap = manifest.uploadedCustomMetadataMap;
             this.codecVersion = manifest.codecVersion;
             this.indices = new ArrayList<>(manifest.indices);
             this.previousClusterUUID = manifest.previousClusterUUID;
@@ -496,18 +672,41 @@ public ClusterMetadataManifest build() {
                 globalMetadataFileName,
                 indices,
                 previousClusterUUID,
-                clusterUUIDCommitted
+                clusterUUIDCommitted,
+                coordinationMetadata,
+                settingsMetadata,
+                templatesMetadata,
+                customMetadataMap
             );
         }
 
     }
 
+    /**
+     * Interface representing uploaded metadata
+     */
+    public interface UploadedMetadata {
+        /**
+         * Gets the component or part of the system this upload belongs to.
+         *
+         * @return A string identifying the component
+         */
+        String getComponent();
+
+        /**
+         * Gets the name of the file that was uploaded
+         *
+         * @return The name of the uploaded file as a string
+         */
+        String getUploadedFilename();
+    }
+
     /**
      * Metadata for uploaded index metadata
      *
      * @opensearch.internal
      */
-    public static class UploadedIndexMetadata implements Writeable, ToXContentFragment {
+    public static class UploadedIndexMetadata implements UploadedMetadata, Writeable, ToXContentFragment {
 
         private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name");
         private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid");
@@ -536,6 +735,7 @@ private static String uploadedFilename(Object[] fields) {
             PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
         }
 
+        static final String COMPONENT_PREFIX = "index--";
         private final String indexName;
         private final String indexUUID;
         private final String uploadedFilename;
@@ -556,6 +756,11 @@ public String getUploadedFilePath() {
             return uploadedFilename;
         }
 
+        @Override
+        public String getComponent() {
+            return COMPONENT_PREFIX + getIndexName();
+        }
+
         public String getUploadedFilename() {
             String[] splitPath = uploadedFilename.split("/");
             return splitPath[splitPath.length - 1];
@@ -613,4 +818,83 @@ public static UploadedIndexMetadata fromXContent(XContentParser parser) throws I
             return PARSER.parse(parser, null);
         }
     }
+
+    /**
+     * Metadata for uploaded metadata attribute
+     *
+     * @opensearch.internal
+     */
+    public static class UploadedMetadataAttribute implements UploadedMetadata, Writeable, ToXContentFragment {
+        private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename");
+
+        private static final ObjectParser.NamedObjectParser<UploadedMetadataAttribute, Void> PARSER;
+
+        static {
+            ConstructingObjectParser<UploadedMetadataAttribute, String> innerParser = new ConstructingObjectParser<>(
+                "uploaded_metadata_attribute",
+                true,
+                (Object[] parsedObject, String name) -> {
+                    String uploadedFilename = (String) parsedObject[0];
+                    return new UploadedMetadataAttribute(name, uploadedFilename);
+                }
+            );
+            innerParser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
+            PARSER = ((p, c, name) -> innerParser.parse(p, name));
+        }
+
+        private final String attributeName;
+        private final String uploadedFilename;
+
+        public UploadedMetadataAttribute(String attributeName, String uploadedFilename) {
+            this.attributeName = attributeName;
+            this.uploadedFilename = uploadedFilename;
+        }
+
+        public UploadedMetadataAttribute(StreamInput in) throws IOException {
+            this.attributeName = in.readString();
+            this.uploadedFilename = in.readString();
+        }
+
+        public String getAttributeName() {
+            return attributeName;
+        }
+
+        @Override
+        public String getComponent() {
+            return getAttributeName();
+        }
+
+        public String getUploadedFilename() {
+            return uploadedFilename;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(attributeName);
+            out.writeString(uploadedFilename);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject(getAttributeName())
+                .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilename())
+                .endObject();
+        }
+
+        public static UploadedMetadataAttribute fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null, parser.currentName());
+        }
+
+        @Override
+        public String toString() {
+            return "UploadedMetadataAttribute{"
+                + "attributeName='"
+                + attributeName
+                + '\''
+                + ", uploadedFilename='"
+                + uploadedFilename
+                + '\''
+                + '}';
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index eaf607564185c..ac821cd15a5b3 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -14,8 +14,11 @@
 import org.opensearch.Version;
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.coordination.CoordinationMetadata;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.metadata.TemplatesMetadata;
+import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobMetadata;
@@ -27,9 +30,9 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
+import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
 import org.opensearch.node.Node;
@@ -63,6 +66,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static java.util.Objects.requireNonNull;
 import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
 
@@ -80,6 +84,7 @@ public class RemoteClusterStateService implements Closeable {
     public static final int RETAINED_MANIFESTS = 10;
 
     public static final String DELIMITER = "__";
+    public static final String CUSTOM_DELIMITER = "--";
 
     private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class);
 
@@ -122,6 +127,30 @@ public class RemoteClusterStateService implements Closeable {
         Metadata::fromXContent
     );
 
+    public static final ChecksumBlobStoreFormat<CoordinationMetadata> COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
+        "coordination",
+        METADATA_NAME_FORMAT,
+        CoordinationMetadata::fromXContent
+    );
+
+    public static final ChecksumBlobStoreFormat<Settings> SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
+        "settings",
+        METADATA_NAME_FORMAT,
+        Settings::fromXContent
+    );
+
+    public static final ChecksumBlobStoreFormat<TemplatesMetadata> TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
+        "templates",
+        METADATA_NAME_FORMAT,
+        TemplatesMetadata::fromXContent
+    );
+
+    public static final ChecksumBlobStoreFormat<Metadata.Custom> CUSTOM_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
+        "custom",
+        METADATA_NAME_FORMAT,
+        Metadata.Custom::fromXContent
+    );
+
     /**
      * Manifest format compatible with older codec v0, where codec version was missing.
      */
@@ -129,7 +158,13 @@ public class RemoteClusterStateService implements Closeable {
         new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0);
 
     /**
-     * Manifest format compatible with codec v1, where we introduced codec versions/global metadata.
+     * Manifest format compatible with older codec v1, where codec versions/global metadata was introduced.
+     */
+    public static final ChecksumBlobStoreFormat<ClusterMetadataManifest> CLUSTER_METADATA_MANIFEST_FORMAT_V1 =
+        new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1);
+
+    /**
+     * Manifest format compatible with codec v2, where global metadata file is replaced with multiple metadata attribute files
      */
     public static final ChecksumBlobStoreFormat<ClusterMetadataManifest> CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>(
         "cluster-metadata-manifest",
@@ -153,6 +188,10 @@ public class RemoteClusterStateService implements Closeable {
     public static final String MANIFEST_PATH_TOKEN = "manifest";
     public static final String MANIFEST_FILE_PREFIX = "manifest";
     public static final String METADATA_FILE_PREFIX = "metadata";
+    public static final String COORDINATION_METADATA = "coordination";
+    public static final String SETTING_METADATA = "settings";
+    public static final String TEMPLATES_METADATA = "templates";
+    public static final String CUSTOM_METADATA = "custom";
     public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion
 
     private final String nodeId;
@@ -171,9 +210,13 @@ public class RemoteClusterStateService implements Closeable {
 
     private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false);
     private final RemotePersistenceStats remoteStateStats;
+    private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]";
+    private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged "
+        + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata "
+        + "updated : [{}], custom metadata updated : [{}]";
     public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1;
-    public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1;
-    public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1;
+    public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2;
+    public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2;
 
     // ToXContent Params with gateway mode.
     // We are using gateway context mode to persist all custom metadata.
@@ -233,22 +276,23 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri
             return null;
         }
 
-        // TODO: we can upload global metadata and index metadata in parallel. [issue: #10645]
-        // Write globalMetadata
-        String globalMetadataFile = writeGlobalMetadata(clusterState);
-
-        List<IndexMetadata> toUpload = new ArrayList<>(clusterState.metadata().indices().values());
-        // any validations before/after upload ?
-        final List<UploadedIndexMetadata> allUploadedIndexMetadata = writeIndexMetadataParallel(
+        UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel(
             clusterState,
-            toUpload,
-            Collections.emptyMap()
+            new ArrayList<>(clusterState.metadata().indices().values()),
+            Collections.emptyMap(),
+            clusterState.metadata().customs(),
+            true,
+            true,
+            true
         );
         final ClusterMetadataManifest manifest = uploadManifest(
             clusterState,
-            allUploadedIndexMetadata,
+            uploadedMetadataResults.uploadedIndexMetadata,
             previousClusterUUID,
-            globalMetadataFile,
+            uploadedMetadataResults.uploadedCoordinationMetadata,
+            uploadedMetadataResults.uploadedSettingsMetadata,
+            uploadedMetadataResults.uploadedTemplatesMetadata,
+            uploadedMetadataResults.uploadedCustomMetadataMap,
             false
         );
         final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos);
@@ -259,13 +303,13 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri
                 "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " + "wrote full state with [{}] indices",
                 durationMillis,
                 slowWriteLoggingThreshold,
-                allUploadedIndexMetadata.size()
+                uploadedMetadataResults.uploadedIndexMetadata.size()
             );
         } else {
             logger.info(
                 "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata",
                 durationMillis,
-                allUploadedIndexMetadata.size()
+                uploadedMetadataResults.uploadedIndexMetadata.size()
             );
         }
         return manifest;
@@ -291,26 +335,15 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         }
         assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term();
 
-        // Write Global Metadata
-        final boolean updateGlobalMetadata = Metadata.isGlobalStateEquals(
-            previousClusterState.metadata(),
-            clusterState.metadata()
-        ) == false;
-        String globalMetadataFile;
-        // For migration case from codec V0 to V1, we have added null check on global metadata file,
-        // If file is empty and codec is 1 then write global metadata.
-        if (updateGlobalMetadata || previousManifest.getGlobalMetadataFileName() == null) {
-            globalMetadataFile = writeGlobalMetadata(clusterState);
-        } else {
-            logger.debug("Global metadata has not updated in cluster state, skipping upload of it");
-            globalMetadataFile = previousManifest.getGlobalMetadataFileName();
+        final Map<String, UploadedMetadataAttribute> customsToBeDeletedFromRemote = new HashMap<>(previousManifest.getCustomMetadataMap());
+        final Map<String, Metadata.Custom> customsToUpload = getUpdatedCustoms(clusterState, previousClusterState);
+        final Map<String, UploadedMetadataAttribute> allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap());
+        for (final String custom : clusterState.metadata().customs().keySet()) {
+            // remove all the customs which are present currently
+            customsToBeDeletedFromRemote.remove(custom);
         }
 
-        // Write Index Metadata
-        final Map<String, IndexMetadata> previousStateIndexMetadataByName = new HashMap<>();
-        for (final IndexMetadata indexMetadata : previousClusterState.metadata().indices().values()) {
-            previousStateIndexMetadataByName.put(indexMetadata.getIndex().getName(), indexMetadata);
-        }
+        final Map<String, IndexMetadata> indicesToBeDeletedFromRemote = new HashMap<>(previousClusterState.metadata().indices());
 
         int numIndicesUpdated = 0;
         int numIndicesUnchanged = 0;
@@ -323,7 +356,7 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         Map<String, IndexMetadata> prevIndexMetadataByName = new HashMap<>();
         for (final IndexMetadata indexMetadata : clusterState.metadata().indices().values()) {
             String indexName = indexMetadata.getIndex().getName();
-            final IndexMetadata prevIndexMetadata = previousStateIndexMetadataByName.get(indexName);
+            final IndexMetadata prevIndexMetadata = indicesToBeDeletedFromRemote.get(indexName);
             Long previousVersion = prevIndexMetadata != null ? prevIndexMetadata.getVersion() : null;
             if (previousVersion == null || indexMetadata.getVersion() != previousVersion) {
                 logger.debug(
@@ -338,22 +371,50 @@ public ClusterMetadataManifest writeIncrementalMetadata(
             } else {
                 numIndicesUnchanged++;
             }
-            previousStateIndexMetadataByName.remove(indexMetadata.getIndex().getName());
+            // index present in current cluster state
+            indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName());
         }
+        UploadedMetadataResults uploadedMetadataResults;
+        // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files,
+        // If file is empty and codec is 1 then write global metadata.
+        boolean firstUploadForSplitGlobalMetadata = !previousManifest.hasMetadataAttributesFiles();
+        boolean updateCoordinationMetadata = firstUploadForSplitGlobalMetadata
+            || Metadata.isCoordinationMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
+        ;
+        boolean updateSettingsMetadata = firstUploadForSplitGlobalMetadata
+            || Metadata.isSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
+        boolean updateTemplatesMetadata = firstUploadForSplitGlobalMetadata
+            || Metadata.isTemplatesMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false;
+
+        uploadedMetadataResults = writeMetadataInParallel(
+            clusterState,
+            toUpload,
+            prevIndexMetadataByName,
+            firstUploadForSplitGlobalMetadata ? clusterState.metadata().customs() : customsToUpload,
+            updateCoordinationMetadata,
+            updateSettingsMetadata,
+            updateTemplatesMetadata
+        );
 
-        List<UploadedIndexMetadata> uploadedIndexMetadataList = writeIndexMetadataParallel(clusterState, toUpload, prevIndexMetadataByName);
-        uploadedIndexMetadataList.forEach(
+        // update the map if the metadata was uploaded
+        uploadedMetadataResults.uploadedIndexMetadata.forEach(
             uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata)
         );
+        allUploadedCustomMap.putAll(uploadedMetadataResults.uploadedCustomMetadataMap);
+        // remove the data for removed custom/indices
+        customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove);
+        indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove);
 
-        for (String removedIndexName : previousStateIndexMetadataByName.keySet()) {
-            allUploadedIndexMetadata.remove(removedIndexName);
-        }
         final ClusterMetadataManifest manifest = uploadManifest(
             clusterState,
             new ArrayList<>(allUploadedIndexMetadata.values()),
             previousManifest.getPreviousClusterUUID(),
-            globalMetadataFile,
+            updateCoordinationMetadata ? uploadedMetadataResults.uploadedCoordinationMetadata : previousManifest.getCoordinationMetadata(),
+            updateSettingsMetadata ? uploadedMetadataResults.uploadedSettingsMetadata : previousManifest.getSettingsMetadata(),
+            updateTemplatesMetadata ? uploadedMetadataResults.uploadedTemplatesMetadata : previousManifest.getTemplatesMetadata(),
+            firstUploadForSplitGlobalMetadata || !customsToUpload.isEmpty()
+                ? allUploadedCustomMap
+                : previousManifest.getCustomMetadataMap(),
             false
         );
         deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS);
@@ -361,115 +422,57 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos);
         remoteStateStats.stateSucceeded();
         remoteStateStats.stateTook(durationMillis);
+        ParameterizedMessage clusterStateUploadTimeMessage = new ParameterizedMessage(
+            CLUSTER_STATE_UPLOAD_TIME_LOG_STRING,
+            manifest.getStateVersion(),
+            durationMillis
+        );
+        ParameterizedMessage metadataUpdateMessage = new ParameterizedMessage(
+            METADATA_UPDATE_LOG_STRING,
+            numIndicesUpdated,
+            numIndicesUnchanged,
+            updateCoordinationMetadata,
+            updateSettingsMetadata,
+            updateTemplatesMetadata,
+            customsToUpload.size()
+        );
         if (durationMillis >= slowWriteLoggingThreshold.getMillis()) {
             logger.warn(
-                "writing cluster state took [{}ms] which is above the warn threshold of [{}]; "
-                    + "wrote  metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]",
-                durationMillis,
+                "{} which is above the warn threshold of [{}]; {}",
+                clusterStateUploadTimeMessage,
                 slowWriteLoggingThreshold,
-                numIndicesUpdated,
-                numIndicesUnchanged,
-                updateGlobalMetadata
+                metadataUpdateMessage
             );
         } else {
-            logger.info(
-                "writing cluster state for version [{}] took [{}ms]; "
-                    + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]",
-                manifest.getStateVersion(),
-                durationMillis,
-                numIndicesUpdated,
-                numIndicesUnchanged,
-                updateGlobalMetadata
-            );
+            logger.info("{}; {}", clusterStateUploadTimeMessage, metadataUpdateMessage);
         }
         return manifest;
     }
 
-    /**
-     * Uploads provided ClusterState's global Metadata to remote store in parallel.
-     * The call is blocking so the method waits for upload to finish and then return.
-     *
-     * @param clusterState current ClusterState
-     * @return String file name where globalMetadata file is stored.
-     */
-    private String writeGlobalMetadata(ClusterState clusterState) throws IOException {
-
-        AtomicReference<String> result = new AtomicReference<String>();
-        AtomicReference<Exception> exceptionReference = new AtomicReference<Exception>();
-
-        final BlobContainer globalMetadataContainer = globalMetadataContainer(
-            clusterState.getClusterName().value(),
-            clusterState.metadata().clusterUUID()
-        );
-        final String globalMetadataFilename = globalMetadataFileName(clusterState.metadata());
-
-        // latch to wait until upload is not finished
-        CountDownLatch latch = new CountDownLatch(1);
-
-        LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> {
-            logger.trace(String.format(Locale.ROOT, "GlobalMetadata uploaded successfully."));
-            result.set(globalMetadataContainer.path().buildAsString() + globalMetadataFilename);
-        }, ex -> { exceptionReference.set(ex); }), latch);
-
-        GLOBAL_METADATA_FORMAT.writeAsyncWithUrgentPriority(
-            clusterState.metadata(),
-            globalMetadataContainer,
-            globalMetadataFilename,
-            blobStoreRepository.getCompressor(),
-            completionListener,
-            FORMAT_PARAMS
-        );
-
-        try {
-            if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
-                // TODO: We should add metrics where transfer is timing out. [Issue: #10687]
-                RemoteStateTransferException ex = new RemoteStateTransferException(
-                    String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete")
-                );
-                throw ex;
-            }
-        } catch (InterruptedException ex) {
-            RemoteStateTransferException exception = new RemoteStateTransferException(
-                String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete - %s"),
-                ex
-            );
-            Thread.currentThread().interrupt();
-            throw exception;
-        }
-        if (exceptionReference.get() != null) {
-            throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get());
-        }
-        return result.get();
-    }
-
-    /**
-     * Uploads provided IndexMetadata's to remote store in parallel. The call is blocking so the method waits for upload to finish and then return.
-     *
-     * @param clusterState current ClusterState
-     * @param toUpload     list of IndexMetadata to upload
-     * @return {@code List<UploadedIndexMetadata>} list of IndexMetadata uploaded to remote
-     */
-    private List<UploadedIndexMetadata> writeIndexMetadataParallel(
+    private UploadedMetadataResults writeMetadataInParallel(
         ClusterState clusterState,
-        List<IndexMetadata> toUpload,
-        Map<String, IndexMetadata> prevIndexMetadataByName
+        List<IndexMetadata> indexToUpload,
+        Map<String, IndexMetadata> prevIndexMetadataByName,
+        Map<String, Metadata.Custom> customToUpload,
+        boolean uploadCoordinationMetadata,
+        boolean uploadSettingsMetadata,
+        boolean uploadTemplateMetadata
     ) throws IOException {
         assert Objects.nonNull(indexMetadataUploadListeners) : "indexMetadataUploadListeners can not be null";
-        int latchCount = toUpload.size() + indexMetadataUploadListeners.size();
-        List<Exception> exceptionList = Collections.synchronizedList(new ArrayList<>(latchCount));
-        final CountDownLatch latch = new CountDownLatch(latchCount);
-        List<UploadedIndexMetadata> result = new ArrayList<>(toUpload.size());
-
-        LatchedActionListener<UploadedIndexMetadata> latchedActionListener = new LatchedActionListener<>(
-            ActionListener.wrap((UploadedIndexMetadata uploadedIndexMetadata) -> {
-                logger.trace(
-                    String.format(Locale.ROOT, "IndexMetadata uploaded successfully for %s", uploadedIndexMetadata.getIndexName())
-                );
-                result.add(uploadedIndexMetadata);
+        int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size()
+            + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0);
+        CountDownLatch latch = new CountDownLatch(totalUploadTasks);
+        Map<String, CheckedRunnable<IOException>> uploadTasks = new HashMap<>(totalUploadTasks);
+        Map<String, ClusterMetadataManifest.UploadedMetadata> results = new HashMap<>(totalUploadTasks);
+        List<Exception> exceptionList = Collections.synchronizedList(new ArrayList<>(totalUploadTasks));
+
+        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> listener = new LatchedActionListener<>(
+            ActionListener.wrap((ClusterMetadataManifest.UploadedMetadata uploadedMetadata) -> {
+                logger.trace(String.format(Locale.ROOT, "Metadata component %s uploaded successfully.", uploadedMetadata.getComponent()));
+                results.put(uploadedMetadata.getComponent(), uploadedMetadata);
             }, ex -> {
-                assert ex instanceof RemoteStateTransferException;
                 logger.error(
-                    () -> new ParameterizedMessage("Exception during transfer of IndexMetadata to Remote {}", ex.getMessage()),
+                    () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()),
                     ex
                 );
                 exceptionList.add(ex);
@@ -477,20 +480,68 @@ private List<UploadedIndexMetadata> writeIndexMetadataParallel(
             latch
         );
 
-        for (IndexMetadata indexMetadata : toUpload) {
-            // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX/metadata_4_1690947200
-            writeIndexMetadataAsync(clusterState, indexMetadata, latchedActionListener);
+        if (uploadSettingsMetadata) {
+            uploadTasks.put(
+                SETTING_METADATA,
+                getAsyncMetadataWriteAction(
+                    clusterState,
+                    SETTING_METADATA,
+                    SETTINGS_METADATA_FORMAT,
+                    clusterState.metadata().persistentSettings(),
+                    listener
+                )
+            );
+        }
+        if (uploadCoordinationMetadata) {
+            uploadTasks.put(
+                COORDINATION_METADATA,
+                getAsyncMetadataWriteAction(
+                    clusterState,
+                    COORDINATION_METADATA,
+                    COORDINATION_METADATA_FORMAT,
+                    clusterState.metadata().coordinationMetadata(),
+                    listener
+                )
+            );
+        }
+        if (uploadTemplateMetadata) {
+            uploadTasks.put(
+                TEMPLATES_METADATA,
+                getAsyncMetadataWriteAction(
+                    clusterState,
+                    TEMPLATES_METADATA,
+                    TEMPLATES_METADATA_FORMAT,
+                    clusterState.metadata().templatesMetadata(),
+                    listener
+                )
+            );
+        }
+        customToUpload.forEach((key, value) -> {
+            String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key);
+            uploadTasks.put(
+                customComponent,
+                getAsyncMetadataWriteAction(clusterState, customComponent, CUSTOM_METADATA_FORMAT, value, listener)
+            );
+        });
+        indexToUpload.forEach(indexMetadata -> {
+            uploadTasks.put(indexMetadata.getIndex().getName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener));
+        });
+
+        // start async upload of all required metadata files
+        for (CheckedRunnable<IOException> uploadTask : uploadTasks.values()) {
+            uploadTask.run();
         }
 
-        invokeIndexMetadataUploadListeners(toUpload, prevIndexMetadataByName, latch, exceptionList);
+        invokeIndexMetadataUploadListeners(indexToUpload, prevIndexMetadataByName, latch, exceptionList);
 
         try {
-            if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
+            if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
+                // TODO: We should add metrics where transfer is timing out. [Issue: #10687]
                 RemoteStateTransferException ex = new RemoteStateTransferException(
                     String.format(
                         Locale.ROOT,
-                        "Timed out waiting for transfer of index metadata to complete - %s",
-                        toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining(""))
+                        "Timed out waiting for transfer of following metadata to complete - %s",
+                        String.join(", ", uploadTasks.keySet())
                     )
                 );
                 exceptionList.forEach(ex::addSuppressed);
@@ -501,26 +552,47 @@ private List<UploadedIndexMetadata> writeIndexMetadataParallel(
             RemoteStateTransferException exception = new RemoteStateTransferException(
                 String.format(
                     Locale.ROOT,
-                    "Timed out waiting for transfer of index metadata to complete - %s",
-                    toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining(""))
+                    "Timed out waiting for transfer of metadata to complete - %s",
+                    String.join(", ", uploadTasks.keySet())
                 ),
                 ex
             );
             Thread.currentThread().interrupt();
             throw exception;
         }
-        if (exceptionList.size() > 0) {
+        if (!exceptionList.isEmpty()) {
             RemoteStateTransferException exception = new RemoteStateTransferException(
                 String.format(
                     Locale.ROOT,
-                    "Exception during transfer of IndexMetadata to Remote %s",
-                    toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining(""))
+                    "Exception during transfer of following metadata to Remote - %s",
+                    String.join(", ", uploadTasks.keySet())
                 )
             );
             exceptionList.forEach(exception::addSuppressed);
             throw exception;
         }
-        return result;
+        UploadedMetadataResults response = new UploadedMetadataResults();
+        results.forEach((name, uploadedMetadata) -> {
+            if (name.contains(CUSTOM_METADATA)) {
+                // component name for custom metadata will look like custom--<metadata-attribute>
+                String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1];
+                response.uploadedCustomMetadataMap.put(
+                    custom,
+                    new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename())
+                );
+            } else if (COORDINATION_METADATA.equals(name)) {
+                response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata;
+            } else if (SETTING_METADATA.equals(name)) {
+                response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata;
+            } else if (TEMPLATES_METADATA.equals(name)) {
+                response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata;
+            } else if (name.contains(UploadedIndexMetadata.COMPONENT_PREFIX)) {
+                response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata);
+            } else {
+                throw new IllegalStateException("Unknown metadata component name " + name);
+            }
+        });
+        return response;
     }
 
     /**
@@ -587,11 +659,11 @@ private ActionListener<Void> getIndexMetadataUploadActionListener(
      * @param indexMetadata         {@link IndexMetadata} to upload
      * @param latchedActionListener listener to respond back on after upload finishes
      */
-    private void writeIndexMetadataAsync(
+    private CheckedRunnable<IOException> getIndexMetadataAsyncAction(
         ClusterState clusterState,
         IndexMetadata indexMetadata,
-        LatchedActionListener<UploadedIndexMetadata> latchedActionListener
-    ) throws IOException {
+        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener
+    ) {
         final BlobContainer indexMetadataContainer = indexMetadataContainer(
             clusterState.getClusterName().value(),
             clusterState.metadata().clusterUUID(),
@@ -609,7 +681,7 @@ private void writeIndexMetadataAsync(
             ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex))
         );
 
-        INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority(
+        return () -> INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority(
             indexMetadata,
             indexMetadataContainer,
             indexMetadataFilename,
@@ -619,6 +691,36 @@ private void writeIndexMetadataAsync(
         );
     }
 
+    /**
+     * Allows async upload of Metadata components to remote
+     */
+
+    private CheckedRunnable<IOException> getAsyncMetadataWriteAction(
+        ClusterState clusterState,
+        String component,
+        ChecksumBlobStoreFormat componentMetadataBlobStore,
+        ToXContent componentMetadata,
+        LatchedActionListener<ClusterMetadataManifest.UploadedMetadata> latchedActionListener
+    ) {
+        final BlobContainer globalMetadataContainer = globalMetadataContainer(
+            clusterState.getClusterName().value(),
+            clusterState.metadata().clusterUUID()
+        );
+        final String componentMetadataFilename = metadataAttributeFileName(component, clusterState.metadata().version());
+        ActionListener<Void> completionListener = ActionListener.wrap(
+            resp -> latchedActionListener.onResponse(new UploadedMetadataAttribute(component, componentMetadataFilename)),
+            ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex))
+        );
+        return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority(
+            componentMetadata,
+            globalMetadataContainer,
+            componentMetadataFilename,
+            blobStoreRepository.getCompressor(),
+            completionListener,
+            FORMAT_PARAMS
+        );
+    }
+
     @Nullable
     public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest)
         throws IOException {
@@ -632,7 +734,10 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat
             clusterState,
             previousManifest.getIndices(),
             previousManifest.getPreviousClusterUUID(),
-            previousManifest.getGlobalMetadataFileName(),
+            previousManifest.getCoordinationMetadata(),
+            previousManifest.getSettingsMetadata(),
+            previousManifest.getTemplatesMetadata(),
+            previousManifest.getCustomMetadataMap(),
             true
         );
         deleteStaleClusterUUIDs(clusterState, committedManifest);
@@ -661,11 +766,19 @@ private ClusterMetadataManifest uploadManifest(
         ClusterState clusterState,
         List<UploadedIndexMetadata> uploadedIndexMetadata,
         String previousClusterUUID,
-        String globalClusterMetadataFileName,
+        UploadedMetadataAttribute uploadedCoordinationMetadata,
+        UploadedMetadataAttribute uploadedSettingsMetadata,
+        UploadedMetadataAttribute uploadedTemplatesMetadata,
+        Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap,
         boolean committed
     ) throws IOException {
         synchronized (this) {
-            final String manifestFileName = getManifestFileName(clusterState.term(), clusterState.version(), committed);
+            final String manifestFileName = getManifestFileName(
+                clusterState.term(),
+                clusterState.version(),
+                committed,
+                MANIFEST_CURRENT_CODEC_VERSION
+            );
             final ClusterMetadataManifest manifest = new ClusterMetadataManifest(
                 clusterState.term(),
                 clusterState.getVersion(),
@@ -675,10 +788,14 @@ private ClusterMetadataManifest uploadManifest(
                 nodeId,
                 committed,
                 MANIFEST_CURRENT_CODEC_VERSION,
-                globalClusterMetadataFileName,
+                null,
                 uploadedIndexMetadata,
                 previousClusterUUID,
-                clusterState.metadata().clusterUUIDCommitted()
+                clusterState.metadata().clusterUUIDCommitted(),
+                uploadedCoordinationMetadata,
+                uploadedSettingsMetadata,
+                uploadedTemplatesMetadata,
+                uploadedCustomMetadataMap
             );
             writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName);
             return manifest;
@@ -699,7 +816,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust
             logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully."));
         }, ex -> { exceptionReference.set(ex); }), latch);
 
-        CLUSTER_METADATA_MANIFEST_FORMAT.writeAsyncWithUrgentPriority(
+        getClusterMetadataManifestBlobStoreFormat(fileName).writeAsyncWithUrgentPriority(
             uploadManifest,
             metadataManifestContainer,
             fileName,
@@ -779,6 +896,31 @@ private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploa
         this.metadataManifestUploadTimeout = newMetadataManifestUploadTimeout;
     }
 
+    private Map<String, Metadata.Custom> getUpdatedCustoms(ClusterState currentState, ClusterState previousState) {
+        if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) {
+            return new HashMap<>();
+        }
+        Map<String, Metadata.Custom> updatedCustom = new HashMap<>();
+        Set<String> currentCustoms = new HashSet<>(currentState.metadata().customs().keySet());
+        for (Map.Entry<String, Metadata.Custom> cursor : previousState.metadata().customs().entrySet()) {
+            if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) {
+                if (currentCustoms.contains(cursor.getKey())
+                    && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) {
+                    // If the custom metadata is updated, we need to upload the new version.
+                    updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey()));
+                }
+                currentCustoms.remove(cursor.getKey());
+            }
+        }
+        for (String custom : currentCustoms) {
+            Metadata.Custom cursor = currentState.metadata().custom(custom);
+            if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) {
+                updatedCustom.put(custom, cursor);
+            }
+        }
+        return updatedCustom;
+    }
+
     public TimeValue getIndexMetadataUploadTimeout() {
         return this.indexMetadataUploadTimeout;
     }
@@ -791,7 +933,7 @@ public TimeValue getMetadataManifestUploadTimeout() {
         return this.metadataManifestUploadTimeout;
     }
 
-    static String getManifestFileName(long term, long version, boolean committed) {
+    static String getManifestFileName(long term, long version, boolean committed, int codecVersion) {
         // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest__<inverted_term>__<inverted_version>__C/P__<inverted__timestamp>__<codec_version>
         return String.join(
             DELIMITER,
@@ -800,7 +942,7 @@ static String getManifestFileName(long term, long version, boolean committed) {
             RemoteStoreUtils.invertLong(version),
             (committed ? "C" : "P"), // C for committed and P for published
             RemoteStoreUtils.invertLong(System.currentTimeMillis()),
-            String.valueOf(MANIFEST_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last place to
+            String.valueOf(codecVersion) // Keep the codec version at last place only, during read we reads last place to
             // determine codec version.
         );
     }
@@ -829,6 +971,17 @@ private static String globalMetadataFileName(Metadata metadata) {
         );
     }
 
+    private static String metadataAttributeFileName(String componentPrefix, Long metadataVersion) {
+        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/<componentPrefix>__<inverted_metadata_version>__<inverted__timestamp>__<codec_version>
+        return String.join(
+            DELIMITER,
+            componentPrefix,
+            RemoteStoreUtils.invertLong(metadataVersion),
+            RemoteStoreUtils.invertLong(System.currentTimeMillis()),
+            String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION)
+        );
+    }
+
     private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) {
         return getCusterMetadataBasePath(clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN);
     }
@@ -895,6 +1048,7 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID
                 String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID)
             );
         }
+
         // Fetch Global Metadata
         Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get());
 
@@ -921,6 +1075,34 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust
                     splitPath[splitPath.length - 1],
                     blobStoreRepository.getNamedXContentRegistry()
                 );
+            } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) {
+                CoordinationMetadata coordinationMetadata = getCoordinationMetadata(
+                    clusterName,
+                    clusterUUID,
+                    clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()
+                );
+                Settings settingsMetadata = getSettingsMetadata(
+                    clusterName,
+                    clusterUUID,
+                    clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()
+                );
+                TemplatesMetadata templatesMetadata = getTemplatesMetadata(
+                    clusterName,
+                    clusterUUID,
+                    clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()
+                );
+                Metadata.Builder builder = new Metadata.Builder();
+                builder.coordinationMetadata(coordinationMetadata);
+                builder.persistentSettings(settingsMetadata);
+                builder.templates(templatesMetadata);
+                clusterMetadataManifest.getCustomMetadataMap()
+                    .forEach(
+                        (key, value) -> builder.putCustom(
+                            key,
+                            getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key)
+                        )
+                    );
+                return builder.build();
             } else {
                 return Metadata.EMPTY_METADATA;
             }
@@ -932,6 +1114,92 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust
         }
     }
 
+    private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) {
+        try {
+            // Fetch Coordination metadata
+            if (coordinationMetadataFileName != null) {
+                String[] splitPath = coordinationMetadataFileName.split("/");
+                return COORDINATION_METADATA_FORMAT.read(
+                    globalMetadataContainer(clusterName, clusterUUID),
+                    splitPath[splitPath.length - 1],
+                    blobStoreRepository.getNamedXContentRegistry()
+                );
+            } else {
+                return CoordinationMetadata.EMPTY_METADATA;
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                String.format(Locale.ROOT, "Error while downloading Coordination Metadata - %s", coordinationMetadataFileName),
+                e
+            );
+        }
+    }
+
+    private Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) {
+        try {
+            // Fetch Settings metadata
+            if (settingsMetadataFileName != null) {
+                String[] splitPath = settingsMetadataFileName.split("/");
+                return SETTINGS_METADATA_FORMAT.read(
+                    globalMetadataContainer(clusterName, clusterUUID),
+                    splitPath[splitPath.length - 1],
+                    blobStoreRepository.getNamedXContentRegistry()
+                );
+            } else {
+                return Settings.EMPTY;
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                String.format(Locale.ROOT, "Error while downloading Settings Metadata - %s", settingsMetadataFileName),
+                e
+            );
+        }
+    }
+
+    private TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) {
+        try {
+            // Fetch Templates metadata
+            if (templatesMetadataFileName != null) {
+                String[] splitPath = templatesMetadataFileName.split("/");
+                return TEMPLATES_METADATA_FORMAT.read(
+                    globalMetadataContainer(clusterName, clusterUUID),
+                    splitPath[splitPath.length - 1],
+                    blobStoreRepository.getNamedXContentRegistry()
+                );
+            } else {
+                return TemplatesMetadata.EMPTY_METADATA;
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", templatesMetadataFileName),
+                e
+            );
+        }
+    }
+
+    private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) {
+        requireNonNull(customMetadataFileName);
+        try {
+            // Fetch Custom metadata
+            String[] splitPath = customMetadataFileName.split("/");
+            ChecksumBlobStoreFormat<Metadata.Custom> customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>(
+                "custom",
+                METADATA_NAME_FORMAT,
+                (parser -> Metadata.Custom.fromXContent(parser, custom))
+            );
+            return customChecksumBlobStoreFormat.read(
+                globalMetadataContainer(clusterName, clusterUUID),
+                splitPath[splitPath.length - 1],
+                blobStoreRepository.getNamedXContentRegistry()
+            );
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName),
+                e
+            );
+        }
+    }
+
     /**
      * Fetch latest ClusterMetadataManifest from remote state store
      *
@@ -1175,6 +1443,8 @@ private ChecksumBlobStoreFormat<ClusterMetadataManifest> getClusterMetadataManif
         long codecVersion = getManifestCodecVersion(fileName);
         if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) {
             return CLUSTER_METADATA_MANIFEST_FORMAT;
+        } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) {
+            return CLUSTER_METADATA_MANIFEST_FORMAT_V1;
         } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) {
             return CLUSTER_METADATA_MANIFEST_FORMAT_V0;
         }
@@ -1319,7 +1589,15 @@ private void deleteClusterMetadata(
                 );
                 clusterMetadataManifest.getIndices()
                     .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename()));
-                filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName());
+                if (clusterMetadataManifest.getGlobalMetadataFileName() != null) {
+                    filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName());
+                } else {
+                    filesToKeep.add(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename());
+                    filesToKeep.add(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename());
+                    filesToKeep.add(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename());
+                    clusterMetadataManifest.getCustomMetadataMap()
+                        .forEach((key, value) -> { filesToKeep.add(value.getUploadedFilename()); });
+                }
             });
             staleManifestBlobMetadata.forEach(blobMetadata -> {
                 ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest(
@@ -1328,14 +1606,56 @@ private void deleteClusterMetadata(
                     blobMetadata.name()
                 );
                 staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name());
-                if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) {
-                    String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
-                    staleGlobalMetadataPaths.add(
-                        new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
-                            globalMetadataSplitPath[globalMetadataSplitPath.length - 1]
-                        )
-                    );
+                if (clusterMetadataManifest.getGlobalMetadataFileName() != null) {
+                    if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) {
+                        String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
+                        staleGlobalMetadataPaths.add(
+                            new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
+                                globalMetadataSplitPath[globalMetadataSplitPath.length - 1]
+                            )
+                        );
+                    }
+                } else {
+                    if (filesToKeep.contains(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()) == false) {
+                        String[] coordinationMetadataSplitPath = clusterMetadataManifest.getCoordinationMetadata()
+                            .getUploadedFilename()
+                            .split("/");
+                        staleGlobalMetadataPaths.add(
+                            new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
+                                coordinationMetadataSplitPath[coordinationMetadataSplitPath.length - 1]
+                            )
+                        );
+                    }
+                    if (filesToKeep.contains(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()) == false) {
+                        String[] templatesMetadataSplitPath = clusterMetadataManifest.getTemplatesMetadata()
+                            .getUploadedFilename()
+                            .split("/");
+                        staleGlobalMetadataPaths.add(
+                            new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
+                                templatesMetadataSplitPath[templatesMetadataSplitPath.length - 1]
+                            )
+                        );
+                    }
+                    if (filesToKeep.contains(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()) == false) {
+                        String[] settingsMetadataSplitPath = clusterMetadataManifest.getSettingsMetadata().getUploadedFilename().split("/");
+                        staleGlobalMetadataPaths.add(
+                            new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
+                                settingsMetadataSplitPath[settingsMetadataSplitPath.length - 1]
+                            )
+                        );
+                    }
+                    clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> {
+                        if (filesToKeep.contains(value.getUploadedFilename()) == false) {
+                            String[] customMetadataSplitPath = value.getUploadedFilename().split("/");
+                            staleGlobalMetadataPaths.add(
+                                new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
+                                    customMetadataSplitPath[customMetadataSplitPath.length - 1]
+                                )
+                            );
+                        }
+                    });
                 }
+
                 clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> {
                     if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) {
                         staleIndexMetadataPaths.add(
@@ -1397,4 +1717,34 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa
     public RemotePersistenceStats getStats() {
         return remoteStateStats;
     }
+
+    private static class UploadedMetadataResults {
+        List<UploadedIndexMetadata> uploadedIndexMetadata;
+        Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap;
+        UploadedMetadataAttribute uploadedCoordinationMetadata;
+        UploadedMetadataAttribute uploadedSettingsMetadata;
+        UploadedMetadataAttribute uploadedTemplatesMetadata;
+
+        public UploadedMetadataResults(
+            List<UploadedIndexMetadata> uploadedIndexMetadata,
+            Map<String, UploadedMetadataAttribute> uploadedCustomMetadataMap,
+            UploadedMetadataAttribute uploadedCoordinationMetadata,
+            UploadedMetadataAttribute uploadedSettingsMetadata,
+            UploadedMetadataAttribute uploadedTemplatesMetadata
+        ) {
+            this.uploadedIndexMetadata = uploadedIndexMetadata;
+            this.uploadedCustomMetadataMap = uploadedCustomMetadataMap;
+            this.uploadedCoordinationMetadata = uploadedCoordinationMetadata;
+            this.uploadedSettingsMetadata = uploadedSettingsMetadata;
+            this.uploadedTemplatesMetadata = uploadedTemplatesMetadata;
+        }
+
+        public UploadedMetadataResults() {
+            this.uploadedIndexMetadata = new ArrayList<>();
+            this.uploadedCustomMetadataMap = new HashMap<>();
+            this.uploadedCoordinationMetadata = null;
+            this.uploadedSettingsMetadata = null;
+            this.uploadedTemplatesMetadata = null;
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java
index 1c0dc7fc1ca2d..bd71aecf89101 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java
@@ -930,20 +930,20 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep
         final VotingConfiguration initialConfig = VotingConfiguration.of(node1);
         final ClusterState clusterState = clusterState(0L, 0L, node1, initialConfig, initialConfig, 42L);
         final String previousClusterUUID = "prev-cluster-uuid";
-        final ClusterMetadataManifest manifest = new ClusterMetadataManifest(
-            0L,
-            0L,
-            randomAlphaOfLength(10),
-            randomAlphaOfLength(10),
-            Version.CURRENT,
-            randomAlphaOfLength(10),
-            false,
-            1,
-            randomAlphaOfLength(10),
-            Collections.emptyList(),
-            randomAlphaOfLength(10),
-            true
-        );
+        final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder()
+            .clusterTerm(0L)
+            .stateVersion(0L)
+            .clusterUUID(randomAlphaOfLength(10))
+            .stateUUID(randomAlphaOfLength(10))
+            .opensearchVersion(Version.CURRENT)
+            .nodeId(randomAlphaOfLength(10))
+            .committed(false)
+            .codecVersion(1)
+            .globalMetadataFileName(randomAlphaOfLength(10))
+            .indices(Collections.emptyList())
+            .previousClusterUUID(randomAlphaOfLength(10))
+            .clusterUUIDCommitted(true)
+            .build();
         Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest);
 
         final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry();
diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
index 6c9a3201656d7..0b3cd49140939 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
@@ -9,6 +9,9 @@
 package org.opensearch.gateway.remote;
 
 import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexGraveyard;
+import org.opensearch.cluster.metadata.RepositoriesMetadata;
+import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -16,32 +19,38 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
+import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
 import org.opensearch.test.EqualsHashCodeTestUtils;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0;
+import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1;
 
 public class ClusterMetadataManifestTests extends OpenSearchTestCase {
 
     public void testClusterMetadataManifestXContentV0() throws IOException {
         UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
-        ClusterMetadataManifest originalManifest = new ClusterMetadataManifest(
-            1L,
-            1L,
-            "test-cluster-uuid",
-            "test-state-uuid",
-            Version.CURRENT,
-            "test-node-id",
-            false,
-            ClusterMetadataManifest.CODEC_V0,
-            null,
-            Collections.singletonList(uploadedIndexMetadata),
-            "prev-cluster-uuid",
-            true
-        );
+        ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .clusterUUID("test-cluster-uuid")
+            .stateUUID("test-state-uuid")
+            .opensearchVersion(Version.CURRENT)
+            .nodeId("test-node-id")
+            .committed(false)
+            .codecVersion(CODEC_V0)
+            .indices(Collections.singletonList(uploadedIndexMetadata))
+            .previousClusterUUID("prev-cluster-uuid")
+            .clusterUUIDCommitted(true)
+            .build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
         originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -53,6 +62,33 @@ public void testClusterMetadataManifestXContentV0() throws IOException {
         }
     }
 
+    public void testClusterMetadataManifestXContentV1() throws IOException {
+        UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
+        ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .clusterUUID("test-cluster-uuid")
+            .stateUUID("test-state-uuid")
+            .opensearchVersion(Version.CURRENT)
+            .nodeId("test-node-id")
+            .committed(false)
+            .codecVersion(CODEC_V1)
+            .globalMetadataFileName("test-global-metadata-file")
+            .indices(Collections.singletonList(uploadedIndexMetadata))
+            .previousClusterUUID("prev-cluster-uuid")
+            .clusterUUIDCommitted(true)
+            .build();
+        final XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+            final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV1(parser);
+            assertEquals(originalManifest, fromXContentManifest);
+        }
+    }
+
     public void testClusterMetadataManifestXContent() throws IOException {
         UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
         ClusterMetadataManifest originalManifest = new ClusterMetadataManifest(
@@ -63,11 +99,31 @@ public void testClusterMetadataManifestXContent() throws IOException {
             Version.CURRENT,
             "test-node-id",
             false,
-            ClusterMetadataManifest.CODEC_V1,
-            "test-global-metadata-file",
+            ClusterMetadataManifest.CODEC_V2,
+            null,
             Collections.singletonList(uploadedIndexMetadata),
             "prev-cluster-uuid",
-            true
+            true,
+            new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"),
+            new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"),
+            new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"),
+            Collections.unmodifiableList(
+                Arrays.asList(
+                    new UploadedMetadataAttribute(
+                        RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE,
+                        "custom--repositories-file"
+                    ),
+                    new UploadedMetadataAttribute(
+                        RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE,
+                        "custom--index_graveyard-file"
+                    ),
+                    new UploadedMetadataAttribute(
+                        RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER
+                            + WeightedRoutingMetadata.TYPE,
+                        "custom--weighted_routing_netadata-file"
+                    )
+                )
+            ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()))
         );
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
@@ -89,11 +145,31 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() {
             Version.CURRENT,
             "B10RX1f5RJenMQvYccCgSQ",
             true,
-            1,
-            "test-global-metadata-file",
+            2,
+            null,
             randomUploadedIndexMetadataList(),
             "yfObdx8KSMKKrXf8UyHhM",
-            true
+            true,
+            new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"),
+            new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"),
+            new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"),
+            Collections.unmodifiableList(
+                Arrays.asList(
+                    new UploadedMetadataAttribute(
+                        RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE,
+                        "custom--repositories-file"
+                    ),
+                    new UploadedMetadataAttribute(
+                        RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE,
+                        "custom--index_graveyard-file"
+                    ),
+                    new UploadedMetadataAttribute(
+                        RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER
+                            + WeightedRoutingMetadata.TYPE,
+                        "custom--weighted_routing_netadata-file"
+                    )
+                )
+            ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()))
         );
         {  // Mutate Cluster Term
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 9f321cd62847c..1b242b921c0d7 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -15,7 +15,9 @@
 import org.opensearch.cluster.coordination.CoordinationMetadata;
 import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.IndexTemplateMetadata;
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.metadata.TemplatesMetadata;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.BlobContainer;
@@ -38,6 +40,7 @@
 import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
+import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
 import org.opensearch.index.remote.RemoteIndexPathUploader;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.indices.IndicesModule;
@@ -47,6 +50,7 @@
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.repositories.fs.FsRepository;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.TestCustomMetadata;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -60,24 +64,29 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 
 import static java.util.stream.Collectors.toList;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION;
@@ -85,11 +94,14 @@
 import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
@@ -227,6 +239,11 @@ public void testWriteFullMetadataSuccess() throws IOException {
         assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
         assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID()));
+        assertThat(manifest.getGlobalMetadataFileName(), nullValue());
+        assertThat(manifest.getCoordinationMetadata(), notNullValue());
+        assertThat(manifest.getSettingsMetadata(), notNullValue());
+        assertThat(manifest.getTemplatesMetadata(), notNullValue());
+        assertFalse(manifest.getCustomMetadataMap().isEmpty());
     }
 
     public void testWriteFullMetadataInParallelSuccess() throws IOException {
@@ -235,16 +252,11 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
 
         ArgumentCaptor<ActionListener<Void>> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
         ArgumentCaptor<WriteContext> writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class);
-        AtomicReference<WriteContext> capturedWriteContext = new AtomicReference<>();
+        ConcurrentHashMap<String, WriteContext> capturedWriteContext = new ConcurrentHashMap<>();
         doAnswer((i) -> {
             actionListenerArgumentCaptor.getValue().onResponse(null);
-            return null;
-        }).doAnswer((i) -> {
-            actionListenerArgumentCaptor.getValue().onResponse(null);
-            capturedWriteContext.set(writeContextArgumentCaptor.getValue());
-            return null;
-        }).doAnswer((i) -> {
-            actionListenerArgumentCaptor.getValue().onResponse(null);
+            WriteContext writeContext = writeContextArgumentCaptor.getValue();
+            capturedWriteContext.put(writeContext.getFileName().split(DELIMITER)[0], writeContextArgumentCaptor.getValue());
             return null;
         }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture());
 
@@ -267,37 +279,41 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
         assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName()));
         assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID()));
         assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue());
-        assertThat(manifest.getGlobalMetadataFileName(), notNullValue());
+        assertThat(manifest.getGlobalMetadataFileName(), nullValue());
+        assertThat(manifest.getCoordinationMetadata(), notNullValue());
+        assertThat(manifest.getSettingsMetadata(), notNullValue());
+        assertThat(manifest.getTemplatesMetadata(), notNullValue());
+        assertThat(manifest.getCustomMetadataMap().size(), not(0));
         assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm()));
         assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
         assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
         assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID()));
 
-        assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 3);
-        assertEquals(writeContextArgumentCaptor.getAllValues().size(), 3);
+        assertEquals(7, actionListenerArgumentCaptor.getAllValues().size());
+        assertEquals(7, writeContextArgumentCaptor.getAllValues().size());
 
-        byte[] writtenBytes = capturedWriteContext.get()
+        byte[] writtenBytes = capturedWriteContext.get("metadata")
             .getStreamProvider(Integer.MAX_VALUE)
             .provideStream(0)
             .getInputStream()
             .readAllBytes();
         IndexMetadata writtenIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.deserialize(
-            capturedWriteContext.get().getFileName(),
+            capturedWriteContext.get("metadata").getFileName(),
             blobStoreRepository.getNamedXContentRegistry(),
             new BytesArray(writtenBytes)
         );
 
-        assertEquals(capturedWriteContext.get().getWritePriority(), WritePriority.URGENT);
+        assertEquals(capturedWriteContext.get("metadata").getWritePriority(), WritePriority.URGENT);
         assertEquals(writtenIndexMetadata.getNumberOfShards(), 1);
         assertEquals(writtenIndexMetadata.getNumberOfReplicas(), 0);
         assertEquals(writtenIndexMetadata.getIndex().getName(), "test-index");
         assertEquals(writtenIndexMetadata.getIndex().getUUID(), "index-uuid");
         long expectedChecksum = RemoteTransferContainer.checksumOfChecksum(new ByteArrayIndexInput("metadata-filename", writtenBytes), 8);
-        if (capturedWriteContext.get().doRemoteDataIntegrityCheck()) {
-            assertEquals(capturedWriteContext.get().getExpectedChecksum().longValue(), expectedChecksum);
+        if (capturedWriteContext.get("metadata").doRemoteDataIntegrityCheck()) {
+            assertEquals(capturedWriteContext.get("metadata").getExpectedChecksum().longValue(), expectedChecksum);
         } else {
-            assertEquals(capturedWriteContext.get().getExpectedChecksum(), null);
+            assertEquals(capturedWriteContext.get("metadata").getExpectedChecksum(), null);
         }
 
     }
@@ -355,7 +371,7 @@ public void testTimeoutWhileWritingManifestFile() throws IOException {
             remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10));
         } catch (Exception e) {
             assertTrue(e instanceof RemoteClusterStateService.RemoteStateTransferException);
-            assertTrue(e.getMessage().contains("Timed out waiting for transfer of manifest file to complete"));
+            assertTrue(e.getMessage().contains("Timed out waiting for transfer of following metadata to complete"));
         }
     }
 
@@ -440,13 +456,28 @@ public void testWriteIncrementalMetadataSuccess() throws IOException {
     }
 
     /*
-     * Here we will verify the migration of manifest file from codec V0 and V1.
+     * Here we will verify the migration of manifest file from codec V0.
      *
      * Initially codec version is 0 and global metadata is also null, we will perform index metadata update.
-     * In final manifest codec version should be 1 and
-     * global metadata should be updated, even if it was not changed in this cluster state update
+     * In final manifest codec version should be 2 and have metadata files updated,
+     * even if it was not changed in this cluster state update
      */
-    public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOException {
+    public void testMigrationFromCodecV0ManifestToCodecV2Manifest() throws IOException {
+        verifyCodecMigrationManifest(ClusterMetadataManifest.CODEC_V0);
+    }
+
+    /*
+     * Here we will verify the migration of manifest file from codec V1.
+     *
+     * Initially codec version is 1 and a global metadata file is there, we will perform index metadata update.
+     * In final manifest codec version should be 2 and have metadata files updated,
+     * even if it was not changed in this cluster state update
+     */
+    public void testMigrationFromCodecV1ManifestToCodecV2Manifest() throws IOException {
+        verifyCodecMigrationManifest(ClusterMetadataManifest.CODEC_V1);
+    }
+
+    private void verifyCodecMigrationManifest(int previousCodec) throws IOException {
         mockBlobStoreObjects();
         final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
         final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -466,7 +497,7 @@ public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOExcepti
 
         // previous manifest with codec 0 and null global metadata
         final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
-            .codecVersion(ClusterMetadataManifest.CODEC_V0)
+            .codecVersion(previousCodec)
             .globalMetadataFileName(null)
             .indices(Collections.emptyList())
             .build();
@@ -479,12 +510,28 @@ public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOExcepti
         );
 
         // global metadata is updated
-        assertThat(manifestAfterUpdate.getGlobalMetadataFileName(), notNullValue());
+        assertThat(manifestAfterUpdate.hasMetadataAttributesFiles(), is(true));
         // Manifest file with codec version with 1 is updated.
-        assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V1));
+        assertThat(manifestAfterUpdate.getCodecVersion(), is(MANIFEST_CURRENT_CODEC_VERSION));
     }
 
-    public void testWriteIncrementalGlobalMetadataSuccess() throws IOException {
+    public void testWriteIncrementalGlobalMetadataFromCodecV0Success() throws IOException {
+        final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build();
+
+        verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(previousManifest);
+    }
+
+    public void testWriteIncrementalGlobalMetadataFromCodecV1Success() throws IOException {
+        final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
+            .codecVersion(1)
+            .globalMetadataFileName("global-metadata-file")
+            .indices(Collections.emptyList())
+            .build();
+
+        verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(previousManifest);
+    }
+
+    private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMetadataManifest previousManifest) throws IOException {
         final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
         mockBlobStoreObjects();
         final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
@@ -492,12 +539,6 @@ public void testWriteIncrementalGlobalMetadataSuccess() throws IOException {
             .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
             .build();
 
-        final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
-            .codecVersion(2)
-            .globalMetadataFileName("global-metadata-file")
-            .indices(Collections.emptyList())
-            .build();
-
         remoteClusterStateService.start();
         final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata(
             previousClusterState,
@@ -506,8 +547,8 @@ public void testWriteIncrementalGlobalMetadataSuccess() throws IOException {
         );
 
         final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
+            .codecVersion(2)
             .indices(Collections.emptyList())
-            .globalMetadataFileName("mock-filename")
             .clusterTerm(1L)
             .stateVersion(1L)
             .stateUUID("state-uuid")
@@ -515,130 +556,354 @@ public void testWriteIncrementalGlobalMetadataSuccess() throws IOException {
             .previousClusterUUID("prev-cluster-uuid")
             .build();
 
-        assertThat(manifest.getGlobalMetadataFileName(), notNullValue());
-        assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm()));
-        assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
-        assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
-        assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
+        assertNull(manifest.getGlobalMetadataFileName());
+        assertNotNull(manifest.getCoordinationMetadata());
+        assertNotNull(manifest.getSettingsMetadata());
+        assertNotNull(manifest.getTemplatesMetadata());
+        assertNotEquals(0, manifest.getCustomMetadataMap().size());
+
+        assertEquals(expectedManifest.getClusterTerm(), manifest.getClusterTerm());
+        assertEquals(expectedManifest.getStateVersion(), manifest.getStateVersion());
+        assertEquals(expectedManifest.getClusterUUID(), manifest.getClusterUUID());
+        assertEquals(expectedManifest.getStateUUID(), manifest.getStateUUID());
+        assertEquals(expectedManifest.getCodecVersion(), manifest.getCodecVersion());
     }
 
-    /*
-     * Here we will verify index metadata is not uploaded again if change is only in global metadata
-     */
-    public void testGlobalMetadataOnlyUpdated() throws IOException {
-        // setup
-        mockBlobStoreObjects();
-        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
-        final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata).version(randomNonNegativeLong()))
+    public void testCoordinationMetadataOnlyUpdated() throws IOException {
+        // Updating the voting config, as updating the term will upload the full cluster state and other files will also get updated
+        Function<ClusterState, ClusterState> updater = (initialClusterState) -> ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.metadata())
+                    .coordinationMetadata(
+                        CoordinationMetadata.builder(initialClusterState.coordinationMetadata())
+                            .addVotingConfigExclusion(new CoordinationMetadata.VotingConfigExclusion("excludedNodeId", "excludedNodeName"))
+                            .build()
+                    )
+                    .build()
+            )
             .build();
-        final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder()
-            .codecVersion(2)
-            .globalMetadataFileName("global-metadata-file")
-            .indices(Collections.emptyList())
+        verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> {
+            // Verify that index metadata information is same in manifest files
+            assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size());
+            IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> {
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName());
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID());
+                // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata
+                // update
+                assertEquals(
+                    metadataAfterUpdate.getIndices().get(i).getUploadedFilename(),
+                    initialMetadata.getIndices().get(i).getUploadedFilename()
+                );
+            });
+
+            // coordination metadata file would have changed
+            assertFalse(
+                metadataAfterUpdate.getCoordinationMetadata()
+                    .getUploadedFilename()
+                    .equalsIgnoreCase(initialMetadata.getCoordinationMetadata().getUploadedFilename())
+            );
+            // Other files will be equal
+            assertEquals(
+                metadataAfterUpdate.getSettingsMetadata().getUploadedFilename(),
+                initialMetadata.getSettingsMetadata().getUploadedFilename()
+            );
+            assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata());
+            assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap());
+        });
+    }
+
+    public void testSettingsMetadataOnlyUpdated() throws IOException {
+        Function<ClusterState, ClusterState> updater = (initialClusterState) -> ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.metadata()).persistentSettings(Settings.builder().put("foo", "bar").build()).build()
+            )
             .build();
-        remoteClusterStateService.start();
 
-        // Initial cluster state with index.
-        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
-        // Updating remote cluster state with changing index metadata
-        final ClusterMetadataManifest manifestAfterIndexMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
-            initialClusterState,
-            clusterState,
-            initialManifest
-        );
+        verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> {
+            // Verify that index metadata information is same in manifest files
+            assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size());
+            IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> {
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName());
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID());
+                // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata
+                // update
+                assertEquals(
+                    metadataAfterUpdate.getIndices().get(i).getUploadedFilename(),
+                    initialMetadata.getIndices().get(i).getUploadedFilename()
+                );
+            });
 
-        // new cluster state where only global metadata is different
-        Metadata newMetadata = Metadata.builder(clusterState.metadata())
-            .persistentSettings(Settings.builder().put("cluster.blocks.read_only", true).build())
-            .version(randomNonNegativeLong())
+            // setting metadata file would have changed
+            assertFalse(
+                metadataAfterUpdate.getSettingsMetadata()
+                    .getUploadedFilename()
+                    .equalsIgnoreCase(initialMetadata.getSettingsMetadata().getUploadedFilename())
+            );
+            assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata());
+            assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata());
+            assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap());
+        });
+    }
+
+    public void testTemplatesMetadataOnlyUpdated() throws IOException {
+        Function<ClusterState, ClusterState> updater = (initialClusterState) -> ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.metadata())
+                    .templates(
+                        TemplatesMetadata.builder()
+                            .put(
+                                IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3))
+                                    .patterns(Arrays.asList("bar-*", "foo-*"))
+                                    .settings(
+                                        Settings.builder()
+                                            .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5))
+                                            .build()
+                                    )
+                                    .build()
+                            )
+                            .build()
+                    )
+                    .build()
+            )
             .build();
-        ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build();
 
-        // updating remote cluster state with global metadata
-        final ClusterMetadataManifest manifestAfterGlobalMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
-            clusterState,
-            newClusterState,
-            manifestAfterIndexMetadataUpdate
-        );
+        verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> {
+            // Verify that index metadata information is same in manifest files
+            assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size());
+            IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> {
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName());
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID());
+                // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata
+                // update
+                assertEquals(
+                    metadataAfterUpdate.getIndices().get(i).getUploadedFilename(),
+                    initialMetadata.getIndices().get(i).getUploadedFilename()
+                );
+            });
 
-        // Verify that index metadata information is same in manifest files
-        assertThat(manifestAfterIndexMetadataUpdate.getIndices().size(), is(manifestAfterGlobalMetadataUpdate.getIndices().size()));
-        assertThat(
-            manifestAfterIndexMetadataUpdate.getIndices().get(0).getIndexName(),
-            is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getIndexName())
-        );
-        assertThat(
-            manifestAfterIndexMetadataUpdate.getIndices().get(0).getIndexUUID(),
-            is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getIndexUUID())
-        );
+            // template metadata file would have changed
+            assertFalse(
+                metadataAfterUpdate.getTemplatesMetadata()
+                    .getUploadedFilename()
+                    .equalsIgnoreCase(initialMetadata.getTemplatesMetadata().getUploadedFilename())
+            );
+            assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata());
+            assertEquals(metadataAfterUpdate.getSettingsMetadata(), initialMetadata.getSettingsMetadata());
+            assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap());
+        });
+    }
 
-        // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata update
-        assertThat(
-            manifestAfterIndexMetadataUpdate.getIndices().get(0).getUploadedFilename(),
-            is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getUploadedFilename())
-        );
+    public void testCustomMetadataOnlyUpdated() throws IOException {
+        Function<ClusterState, ClusterState> updater = (initialClusterState) -> ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.metadata())
+                    .putCustom("custom_metadata_type", new CustomMetadata1("mock_custom_metadata"))
+                    .build()
+            )
+            .build();
+
+        verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> {
+            // Verify that index metadata information is same in manifest files
+            assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size());
+            IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> {
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName());
+                assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID());
+                // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata
+                // update
+                assertEquals(
+                    metadataAfterUpdate.getIndices().get(i).getUploadedFilename(),
+                    initialMetadata.getIndices().get(i).getUploadedFilename()
+                );
+                // custom metadata map would have changed
+                assertNotEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap());
+                assertEquals(initialMetadata.getCustomMetadataMap().size() + 1, metadataAfterUpdate.getCustomMetadataMap().size());
+                initialMetadata.getCustomMetadataMap().forEach((k, v) -> {
+                    assertTrue(metadataAfterUpdate.getCustomMetadataMap().containsKey(k));
+                    assertEquals(v, metadataAfterUpdate.getCustomMetadataMap().get(k));
+                });
+                assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata());
+                assertEquals(metadataAfterUpdate.getSettingsMetadata(), initialMetadata.getSettingsMetadata());
+                assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata());
+            });
+        });
+    }
 
-        // global metadata file would have changed
-        assertFalse(
-            manifestAfterIndexMetadataUpdate.getGlobalMetadataFileName()
-                .equalsIgnoreCase(manifestAfterGlobalMetadataUpdate.getGlobalMetadataFileName())
+    public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException {
+        // setup
+        mockBlobStoreObjects();
+
+        // Initial cluster state with index.
+        final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_");
+
+        ClusterState clusterState1 = ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.metadata())
+                    .putCustom("custom1", new CustomMetadata1("mock_custom_metadata1"))
+                    .putCustom("custom2", new CustomMetadata1("mock_custom_metadata2"))
+                    .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3"))
+            )
+            .build();
+
+        ClusterMetadataManifest manifest1 = remoteClusterStateService.writeIncrementalMetadata(
+            initialClusterState,
+            clusterState1,
+            initialManifest
         );
+        // remove custom1 from the cluster state, update custom2, custom3 is at it is, added custom4
+        ClusterState clusterState2 = ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.metadata())
+                    .putCustom("custom2", new CustomMetadata1("mock_updated_custom_metadata"))
+                    .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3"))
+                    .putCustom("custom4", new CustomMetadata1("mock_custom_metadata4"))
+            )
+            .build();
+        ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1);
+        // custom1 is removed
+        assertFalse(manifest2.getCustomMetadataMap().containsKey("custom1"));
+        // custom2 is updated
+        assertNotEquals(manifest1.getCustomMetadataMap().get("custom2"), manifest2.getCustomMetadataMap().get("custom2"));
+        // custom3 is unchanged
+        assertEquals(manifest1.getCustomMetadataMap().get("custom3"), manifest2.getCustomMetadataMap().get("custom3"));
+        // custom4 is added
+        assertTrue(manifest2.getCustomMetadataMap().containsKey("custom4"));
+        assertFalse(manifest1.getCustomMetadataMap().containsKey("custom4"));
     }
 
     /*
      * Here we will verify global metadata is not uploaded again if change is only in index metadata
      */
     public void testIndexMetadataOnlyUpdated() throws IOException {
+        Function<ClusterState, ClusterState> updater = (initialState) -> ClusterState.builder(initialState)
+            .metadata(
+                Metadata.builder(initialState.metadata())
+                    .put(
+                        IndexMetadata.builder("test" + randomAlphaOfLength(3))
+                            .settings(
+                                Settings.builder()
+                                    .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                                    .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
+                                    .build()
+                            )
+                            .numberOfShards(1)
+                            .numberOfReplicas(0)
+                    )
+                    .build()
+            )
+            .build();
+
+        verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> {
+            assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata());
+            assertEquals(metadataAfterUpdate.getSettingsMetadata(), initialMetadata.getSettingsMetadata());
+            assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata());
+            assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap());
+            assertEquals(initialMetadata.getIndices().size() + 1, metadataAfterUpdate.getIndices().size());
+        });
+    }
+
+    public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException {
         // setup
         mockBlobStoreObjects();
-        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
-        final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+
+        // Initial cluster state with index.
+        final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_");
+        String initialIndex = "test-index";
+        Index index1 = new Index("test-index-1", "index-uuid-1");
+        Index index2 = new Index("test-index-2", "index-uuid-2");
+        Settings idxSettings1 = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, index1.getUUID())
             .build();
-        final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder()
-            .codecVersion(2)
-            .indices(Collections.emptyList())
+        IndexMetadata indexMetadata1 = new IndexMetadata.Builder(index1.getName()).settings(idxSettings1)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
             .build();
-        remoteClusterStateService.start();
-
-        // Initial cluster state with global metadata.
-        final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
-
-        // Updating remote cluster state with changing global metadata
-        final ClusterMetadataManifest manifestAfterGlobalMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
+        Settings idxSettings2 = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, index2.getUUID())
+            .build();
+        IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        ClusterState clusterState1 = ClusterState.builder(initialClusterState)
+            .metadata(
+                Metadata.builder(initialClusterState.getMetadata())
+                    .put(indexMetadata1, true)
+                    .put(indexMetadata2, true)
+                    .remove(initialIndex)
+                    .build()
+            )
+            .build();
+        ClusterMetadataManifest manifest1 = remoteClusterStateService.writeIncrementalMetadata(
             initialClusterState,
-            clusterState,
+            clusterState1,
             initialManifest
         );
+        // verify that initial index is removed, and new index are added
+        assertEquals(1, initialManifest.getIndices().size());
+        assertEquals(2, manifest1.getIndices().size());
+        assertTrue(initialManifest.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(initialIndex)));
+        assertFalse(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(initialIndex)));
+        // update index1, index2 is unchanged
+        indexMetadata1 = new IndexMetadata.Builder(indexMetadata1).version(indexMetadata1.getVersion() + 1).build();
+        ClusterState clusterState2 = ClusterState.builder(clusterState1)
+            .metadata(Metadata.builder(clusterState1.getMetadata()).put(indexMetadata1, true).build())
+            .build();
+        ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1);
+        // index1 is updated
+        assertEquals(2, manifest2.getIndices().size());
+        assertEquals(
+            1,
+            manifest2.getIndices().stream().filter(uploadedIndex -> uploadedIndex.getIndexName().equals(index1.getName())).count()
+        );
+        assertNotEquals(
+            manifest2.getIndices()
+                .stream()
+                .filter(uploadedIndex -> uploadedIndex.getIndexName().equals(index1.getName()))
+                .findFirst()
+                .get()
+                .getUploadedFilename(),
+            manifest1.getIndices()
+                .stream()
+                .filter(uploadedIndex -> uploadedIndex.getIndexName().equals(index1.getName()))
+                .findFirst()
+                .get()
+                .getUploadedFilename()
+        );
+    }
 
-        // new cluster state where only Index metadata is different
-        final IndexMetadata indexMetadata = new IndexMetadata.Builder("test").settings(
-            Settings.builder()
-                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
-                .build()
-        ).numberOfShards(1).numberOfReplicas(0).build();
-        Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build();
-        ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build();
+    private void verifyMetadataAttributeOnlyUpdated(
+        Function<ClusterState, ClusterState> clusterStateUpdater,
+        BiConsumer<ClusterMetadataManifest, ClusterMetadataManifest> assertions
+    ) throws IOException {
+        // setup
+        mockBlobStoreObjects();
 
-        // updating remote cluster state with index metadata
-        final ClusterMetadataManifest manifestAfterIndexMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
-            clusterState,
-            newClusterState,
-            manifestAfterGlobalMetadataUpdate
-        );
+        // Initial cluster state with index.
+        final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_");
 
-        // Verify that global metadata information is same in manifest files after updating index Metadata
-        // since timestamp is part of file name, if file name is same we can confirm that file is not update in index metadata update
-        assertThat(
-            manifestAfterIndexMetadataUpdate.getGlobalMetadataFileName(),
-            is(manifestAfterGlobalMetadataUpdate.getGlobalMetadataFileName())
-        );
+        ClusterState newClusterState = clusterStateUpdater.apply(initialClusterState);
+
+        // updating remote cluster state with global metadata
+        final ClusterMetadataManifest manifestAfterMetadataUpdate;
+        if (initialClusterState.term() == newClusterState.term()) {
+            manifestAfterMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
+                initialClusterState,
+                newClusterState,
+                initialManifest
+            );
+        } else {
+            manifestAfterMetadataUpdate = remoteClusterStateService.writeFullMetadata(newClusterState, initialClusterState.stateUUID());
+        }
 
-        // Index metadata would have changed
-        assertThat(manifestAfterGlobalMetadataUpdate.getIndices().size(), is(0));
-        assertThat(manifestAfterIndexMetadataUpdate.getIndices().size(), is(1));
+        assertions.accept(initialManifest, manifestAfterMetadataUpdate);
     }
 
     public void testReadLatestMetadataManifestFailedIOException() throws IOException {
@@ -799,7 +1064,10 @@ public void testReadGlobalMetadata() throws IOException {
             .stateUUID("state-uuid")
             .clusterUUID("cluster-uuid")
             .codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
-            .globalMetadataFileName("global-metadata-file")
+            .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file"))
+            .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file"))
+            .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file"))
+            .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file"))
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
@@ -808,7 +1076,7 @@ public void testReadGlobalMetadata() throws IOException {
         Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build();
         mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expactedMetadata);
 
-        ClusterState newClusterState = remoteClusterStateService.getLatestClusterState(
+         ClusterState newClusterState = remoteClusterStateService.getLatestClusterState(
             clusterState.getClusterName().value(),
             clusterState.metadata().clusterUUID()
         );
@@ -834,7 +1102,7 @@ public void testReadGlobalMetadataIOException() throws IOException {
             .stateVersion(1L)
             .stateUUID("state-uuid")
             .clusterUUID("cluster-uuid")
-            .codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
+            .codecVersion(ClusterMetadataManifest.CODEC_V1)
             .globalMetadataFileName(globalIndexMetadataName)
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
@@ -1128,18 +1396,24 @@ public void testFileNames() {
         assertThat(splittedIndexMetadataFileName[1], is(RemoteStoreUtils.invertLong(indexMetadata.getVersion())));
         assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION)));
 
+        verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION);
+        verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1);
+        verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0);
+    }
+
+    private void verifyManifestFileNameWithCodec(int codecVersion) {
         int term = randomIntBetween(5, 10);
         int version = randomIntBetween(5, 10);
-        String manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, true);
+        String manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, true, codecVersion);
         assertThat(manifestFileName.split(DELIMITER).length, is(6));
         String[] splittedName = manifestFileName.split(DELIMITER);
         assertThat(splittedName[0], is(MANIFEST_FILE_PREFIX));
         assertThat(splittedName[1], is(RemoteStoreUtils.invertLong(term)));
         assertThat(splittedName[2], is(RemoteStoreUtils.invertLong(version)));
         assertThat(splittedName[3], is("C"));
-        assertThat(splittedName[5], is(String.valueOf(MANIFEST_CURRENT_CODEC_VERSION)));
+        assertThat(splittedName[5], is(String.valueOf(codecVersion)));
 
-        manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, false);
+        manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, false, codecVersion);
         splittedName = manifestFileName.split(DELIMITER);
         assertThat(splittedName[3], is("P"));
     }
@@ -1255,12 +1529,16 @@ private void mockObjectsForGettingPreviousClusterUUID(
             new UploadedIndexMetadata("index1", "index-uuid1", "key1"),
             new UploadedIndexMetadata("index2", "index-uuid2", "key2")
         );
+        Map<String, UploadedMetadataAttribute> customMetadataMap = new HashMap<>();
         final ClusterMetadataManifest clusterManifest1 = generateClusterMetadataManifest(
             "cluster-uuid1",
             clusterUUIDsPointers.get("cluster-uuid1"),
             randomAlphaOfLength(10),
             uploadedIndexMetadataList1,
-            "test-metadata1",
+            customMetadataMap,
+            new UploadedMetadataAttribute(COORDINATION_METADATA, "key3"),
+            new UploadedMetadataAttribute(SETTING_METADATA, "key4"),
+            new UploadedMetadataAttribute(TEMPLATES_METADATA, "key5"),
             clusterUUIDCommitted.getOrDefault("cluster-uuid1", true)
         );
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
@@ -1279,7 +1557,7 @@ private void mockObjectsForGettingPreviousClusterUUID(
             .build();
         Map<String, IndexMetadata> indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2);
         mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1);
-        mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V1);
+        mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V2);
 
         List<UploadedIndexMetadata> uploadedIndexMetadataList2 = List.of(
             new UploadedIndexMetadata("index1", "index-uuid1", "key1"),
@@ -1290,7 +1568,10 @@ private void mockObjectsForGettingPreviousClusterUUID(
             clusterUUIDsPointers.get("cluster-uuid2"),
             randomAlphaOfLength(10),
             uploadedIndexMetadataList2,
-            "test-metadata2",
+            customMetadataMap,
+            new UploadedMetadataAttribute(COORDINATION_METADATA, "key3"),
+            new UploadedMetadataAttribute(SETTING_METADATA, "key4"),
+            new UploadedMetadataAttribute(TEMPLATES_METADATA, "key5"),
             clusterUUIDCommitted.getOrDefault("cluster-uuid2", true)
         );
         IndexMetadata indexMetadata3 = IndexMetadata.builder("index1")
@@ -1308,7 +1589,7 @@ private void mockObjectsForGettingPreviousClusterUUID(
             .build();
         Map<String, IndexMetadata> indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4);
         mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2);
-        mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V1);
+        mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V2);
 
         // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different
         // when comparing cluster-uuid3 and cluster-uuid1 state.
@@ -1330,17 +1611,19 @@ private void mockObjectsForGettingPreviousClusterUUID(
         Metadata metadata3 = Metadata.builder()
             .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), !differGlobalMetadata).build())
             .build();
-
         final ClusterMetadataManifest clusterManifest3 = generateClusterMetadataManifest(
             "cluster-uuid3",
             clusterUUIDsPointers.get("cluster-uuid3"),
             randomAlphaOfLength(10),
             uploadedIndexMetadataList3,
-            "test-metadata3",
+            customMetadataMap,
+            new UploadedMetadataAttribute(COORDINATION_METADATA, "key3"),
+            new UploadedMetadataAttribute(SETTING_METADATA, "key4"),
+            new UploadedMetadataAttribute(TEMPLATES_METADATA, "key5"),
             clusterUUIDCommitted.getOrDefault("cluster-uuid3", true)
         );
         mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3);
-        mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V1);
+        mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V2);
 
         ArrayList<BlobContainer> mockBlobContainerOrderedList = new ArrayList<>(
             List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2)
@@ -1360,7 +1643,7 @@ private void mockObjectsForGettingPreviousClusterUUID(
         when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor());
     }
 
-    private ClusterMetadataManifest generateClusterMetadataManifest(
+    private ClusterMetadataManifest generateV1ClusterMetadataManifest(
         String clusterUUID,
         String previousClusterUUID,
         String stateUUID,
@@ -1384,6 +1667,36 @@ private ClusterMetadataManifest generateClusterMetadataManifest(
             .build();
     }
 
+    private ClusterMetadataManifest generateClusterMetadataManifest(
+        String clusterUUID,
+        String previousClusterUUID,
+        String stateUUID,
+        List<UploadedIndexMetadata> uploadedIndexMetadata,
+        Map<String, UploadedMetadataAttribute> customMetadataMap,
+        UploadedMetadataAttribute coordinationMetadata,
+        UploadedMetadataAttribute settingsMetadata,
+        UploadedMetadataAttribute templatesMetadata,
+        Boolean isUUIDCommitted
+    ) {
+        return ClusterMetadataManifest.builder()
+            .indices(uploadedIndexMetadata)
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .stateUUID(stateUUID)
+            .clusterUUID(clusterUUID)
+            .nodeId("nodeA")
+            .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
+            .previousClusterUUID(previousClusterUUID)
+            .committed(true)
+            .clusterUUIDCommitted(isUUIDCommitted)
+            .coordinationMetadata(coordinationMetadata)
+            .settingMetadata(settingsMetadata)
+            .templatesMetadata(templatesMetadata)
+            .customMetadataMap(customMetadataMap)
+            .codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
+            .build();
+    }
+
     private BlobContainer mockBlobStoreObjects() {
         return mockBlobStoreObjects(BlobContainer.class);
     }
@@ -1423,7 +1736,7 @@ private void mockBlobContainer(
         int codecVersion
     ) throws IOException {
         String manifestFileName = codecVersion >= ClusterMetadataManifest.CODEC_V1
-            ? "manifest__manifestFileName__abcd__abcd__abcd__1"
+            ? "manifest__manifestFileName__abcd__abcd__abcd__" + codecVersion
             : "manifestFileName";
         BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1);
         when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC))
@@ -1464,7 +1777,8 @@ private void mockBlobContainerForGlobalMetadata(
         ClusterMetadataManifest clusterMetadataManifest,
         Metadata metadata
     ) throws IOException {
-        String mockManifestFileName = "manifest__1__2__C__456__1";
+        int codecVersion = clusterMetadataManifest.getCodecVersion();
+        String mockManifestFileName = "manifest__1__2__C__456__" + codecVersion;
         BlobMetadata blobMetadata = new PlainBlobMetadata(mockManifestFileName, 1);
         when(
             blobContainer.listBlobsByPrefixInSortedOrder(
@@ -1481,19 +1795,84 @@ private void mockBlobContainerForGlobalMetadata(
             FORMAT_PARAMS
         );
         when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
+        if (codecVersion >= ClusterMetadataManifest.CODEC_V2) {
+            String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename());
+            when(blobContainer.readBlob(RemoteClusterStateService.COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer(
+                (invocationOnMock) -> {
+                    BytesReference bytesReference = RemoteClusterStateService.COORDINATION_METADATA_FORMAT.serialize(
+                        metadata.coordinationMetadata(),
+                        coordinationFileName,
+                        blobStoreRepository.getCompressor(),
+                        FORMAT_PARAMS
+                    );
+                    return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes());
+                }
+            );
+
+            String settingsFileName = getFileNameFromPath(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename());
+            when(blobContainer.readBlob(RemoteClusterStateService.SETTINGS_METADATA_FORMAT.blobName(settingsFileName))).thenAnswer(
+                (invocationOnMock) -> {
+                    BytesReference bytesReference = RemoteClusterStateService.SETTINGS_METADATA_FORMAT.serialize(
+                        metadata.persistentSettings(),
+                        settingsFileName,
+                        blobStoreRepository.getCompressor(),
+                        FORMAT_PARAMS
+                    );
+                    return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes());
+                }
+            );
 
-        String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
-        when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))).thenAnswer(
-            (invocationOnMock) -> {
-                BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize(
-                    metadata,
-                    "global-metadata-file",
-                    blobStoreRepository.getCompressor(),
-                    FORMAT_PARAMS
+            String templatesFileName = getFileNameFromPath(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename());
+            when(blobContainer.readBlob(RemoteClusterStateService.TEMPLATES_METADATA_FORMAT.blobName(templatesFileName))).thenAnswer(
+                (invocationOnMock) -> {
+                    BytesReference bytesReference = RemoteClusterStateService.TEMPLATES_METADATA_FORMAT.serialize(
+                        metadata.templatesMetadata(),
+                        templatesFileName,
+                        blobStoreRepository.getCompressor(),
+                        FORMAT_PARAMS
+                    );
+                    return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes());
+                }
+            );
+
+            Map<String, String> customFileMap = clusterMetadataManifest.getCustomMetadataMap()
+                .entrySet()
+                .stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, entry -> getFileNameFromPath(entry.getValue().getUploadedFilename())));
+
+            for (Map.Entry<String, String> entry : customFileMap.entrySet()) {
+                String custom = entry.getKey();
+                String fileName = entry.getValue();
+                when(blobContainer.readBlob(RemoteClusterStateService.CUSTOM_METADATA_FORMAT.blobName(fileName))).thenAnswer(
+                    (invocation) -> {
+                        BytesReference bytesReference = RemoteClusterStateService.CUSTOM_METADATA_FORMAT.serialize(
+                            metadata.custom(custom),
+                            fileName,
+                            blobStoreRepository.getCompressor(),
+                            FORMAT_PARAMS
+                        );
+                        return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes());
+                    }
                 );
-                return new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes());
             }
-        );
+        } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) {
+            String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
+            when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1])))
+                .thenAnswer((invocationOnMock) -> {
+                    BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize(
+                        metadata,
+                        "global-metadata-file",
+                        blobStoreRepository.getCompressor(),
+                        FORMAT_PARAMS
+                    );
+                    return new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes());
+                });
+        }
+    }
+
+    private String getFileNameFromPath(String filePath) {
+        String[] splitPath = filePath.split("/");
+        return splitPath[splitPath.length - 1];
     }
 
     private static ClusterState.Builder generateClusterStateWithGlobalMetadata() {
@@ -1523,7 +1902,9 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() {
             .numberOfReplicas(0)
             .build();
         final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
-
+        final Settings settings = Settings.builder().put("mock-settings", true).build();
+        final TemplatesMetadata templatesMetadata = TemplatesMetadata.EMPTY_METADATA;
+        final CustomMetadata1 customMetadata1 = new CustomMetadata1("custom-metadata-1");
         return ClusterState.builder(ClusterName.DEFAULT)
             .version(1L)
             .stateUUID("state-uuid")
@@ -1533,6 +1914,9 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() {
                     .put(indexMetadata, true)
                     .clusterUUID("cluster-uuid")
                     .coordinationMetadata(coordinationMetadata)
+                    .persistentSettings(settings)
+                    .templates(templatesMetadata)
+                    .putCustom(customMetadata1.getWriteableName(), customMetadata1)
                     .build()
             );
     }
@@ -1541,4 +1925,27 @@ private static DiscoveryNodes nodesWithLocalNodeClusterManager() {
         return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").build();
     }
 
+    private static class CustomMetadata1 extends TestCustomMetadata {
+        public static final String TYPE = "custom_md_1";
+
+        CustomMetadata1(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<Metadata.XContentContext> context() {
+            return EnumSet.of(Metadata.XContentContext.GATEWAY);
+        }
+    }
+
 }

From f21727060dc676046a295db94ef15c6f2b96339d Mon Sep 17 00:00:00 2001
From: Finn <finnegancarroll94@gmail.com>
Date: Thu, 16 May 2024 07:31:38 -0700
Subject: [PATCH 39/52] Enhance tika document parsing tests (#13618)

* Update tika document parsing bwc tests.

Signed-off-by: Carroll <carrofin@amazon.com>

* Skip sample tika files which do not parse consistently.

Signed-off-by: Carroll <carrofin@amazon.com>

* Formatting for spotlessJavaCheck.

Signed-off-by: Carroll <carrofin@amazon.com>

* Use fixed locale for consistent tika parsing.

Signed-off-by: Carroll <carrofin@amazon.com>

* Move sha1 map to .checksums file.

Signed-off-by: Carroll <carrofin@amazon.com>

* For locale dependant files do not verify contents with hash.

Signed-off-by: Carroll <carrofin@amazon.com>

* Remove strict checksum validation for additional locale dependant files.

Signed-off-by: Carroll <carrofin@amazon.com>

---------

Signed-off-by: Carroll <carrofin@amazon.com>
---
 .../ingest/attachment/TikaDocTests.java       |  65 +++---
 .../ingest/attachment/test/.checksums         | 209 ++++++++++++++++++
 2 files changed, 248 insertions(+), 26 deletions(-)
 create mode 100644 plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums

diff --git a/plugins/ingest-attachment/src/test/java/org/opensearch/ingest/attachment/TikaDocTests.java b/plugins/ingest-attachment/src/test/java/org/opensearch/ingest/attachment/TikaDocTests.java
index 7f4a9b8ca0ac7..a022b8b9bf8b0 100644
--- a/plugins/ingest-attachment/src/test/java/org/opensearch/ingest/attachment/TikaDocTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/opensearch/ingest/attachment/TikaDocTests.java
@@ -32,54 +32,67 @@
 
 package org.opensearch.ingest.attachment;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.tika.metadata.Metadata;
 import org.opensearch.common.io.PathUtils;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
 /**
- * Evil test-coverage cheat, we parse a bunch of docs from tika
- * so that we have a nice grab-bag variety, and assert some content
- * comes back and no exception.
+ * Parse sample tika documents and assert the contents has not changed according to previously recorded checksums.
+ * Uncaught changes to tika parsing could potentially pose bwc issues.
+ * Note: In some cases tika will access a user's locale to inform the parsing of a file.
+ * The checksums of these files are left empty, and we only validate that parsed content is not null.
  */
 @SuppressFileSystems("ExtrasFS") // don't try to parse extraN
 public class TikaDocTests extends OpenSearchTestCase {
 
-    /** some test files from tika test suite, zipped up */
+    /** some test files from the apache tika unit test suite with accompanying sha1 checksums */
     static final String TIKA_FILES = "/org/opensearch/ingest/attachment/test/tika-files/";
+    static final String TIKA_CHECKSUMS = "/org/opensearch/ingest/attachment/test/.checksums";
 
-    public void testFiles() throws Exception {
-        Path tmp = createTempDir();
-        logger.debug("unzipping all tika sample files");
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(TIKA_FILES).toURI()))) {
-            for (Path doc : stream) {
-                String filename = doc.getFileName().toString();
-                TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES + filename), tmp);
-            }
-        }
+    public void testParseSamples() throws Exception {
+        String checksumJson = Files.readString(PathUtils.get(getClass().getResource(TIKA_CHECKSUMS).toURI()));
+        Map<String, Object> checksums = XContentHelper.convertToMap(JsonXContent.jsonXContent, checksumJson, false);
+        DirectoryStream<Path> stream = Files.newDirectoryStream(unzipToTemp(TIKA_FILES));
 
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmp)) {
-            for (Path doc : stream) {
-                logger.debug("parsing: {}", doc);
-                assertParseable(doc);
+        for (Path doc : stream) {
+            String parsedContent = tryParse(doc);
+            assertNotNull(parsedContent);
+            assertFalse(parsedContent.isEmpty());
+
+            String check = checksums.get(doc.getFileName().toString()).toString();
+            if (!check.isEmpty()) {
+                assertEquals(check, DigestUtils.sha1Hex(parsedContent));
             }
         }
+
+        stream.close();
     }
 
-    void assertParseable(Path fileName) throws Exception {
-        try {
-            byte bytes[] = Files.readAllBytes(fileName);
-            String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1);
-            assertNotNull(parsedContent);
-            assertFalse(parsedContent.isEmpty());
-            logger.debug("extracted content: {}", parsedContent);
-        } catch (Exception e) {
-            throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e);
+    private Path unzipToTemp(String zipDir) throws Exception {
+        Path tmp = createTempDir();
+        DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(zipDir).toURI()));
+
+        for (Path doc : stream) {
+            String filename = doc.getFileName().toString();
+            TestUtil.unzip(getClass().getResourceAsStream(zipDir + filename), tmp);
         }
+
+        stream.close();
+        return tmp;
+    }
+
+    private String tryParse(Path doc) throws Exception {
+        byte bytes[] = Files.readAllBytes(doc);
+        return TikaImpl.parse(bytes, new Metadata(), -1);
     }
 }
diff --git a/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums b/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums
new file mode 100644
index 0000000000000..227d7d833a231
--- /dev/null
+++ b/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums
@@ -0,0 +1,209 @@
+{
+    "testWORD_tabular_symbol.doc": "c708d7ef841f7e1748436b8ef5670d0b2de1a227",
+    "testWORD_1img.docx": "367e2ade13ca3c19bcd8a323e21d51d407e017ac",
+    "testMasterFooter.odp": "bcc59df70699c739423a50e362c722b81ae76498",
+    "testTXTNonASCIIUTF8.txt": "1ef514431ca8d838f11e99f8e4a0637730b77aa0",
+    "EmbeddedOutlook.docx": "c544a6765c19ba11b0bf3edb55c79e1bd8565c6e",
+    "testWORD_override_list_numbering.docx": "4e892319b921322916225def763f451e4bbb4e16",
+    "testTextBoxes.key": "b01581d5bd2483ce649a1a1406136359f4b93167",
+    "testPPT_masterText.pptx": "9fee8337b76dc3e196f4554dcde22b9dd1c3b3e8",
+    "testComment.docx": "333b9009686f27265b4729e8172b3e62048ec7ec",
+    "testRTFInvalidUnicode.rtf": "32b3e3d8e5c5a1b66cb15fc964b9341bea7048f4",
+    "testEXCEL_headers_footers.xlsx": "9e8d2a700fc431fe29030e86e08162fc8ecf2c1a",
+    "testWORD6.doc": "1479de589755c7212815445799c44dab69d4587c",
+    "testPagesHeadersFootersFootnotes.pages": "99d434be7de4902dc70700aa9c2a31624583c1f1",
+    "testPDF_no_extract_yes_accessibility_owner_empty.pdf": "6eb693dac68fece3bf3cd1aa9880ea9b23fc927c",
+    "testOpenOffice2.odt": "564b3e1999a53073a04142e01b663757a6e7fb08",
+    "testTables.key": "250cff75db7fc3c8b95b2cbd3f37308826e0c93d",
+    "testDOCX_Thumbnail.docx": "fce6a43271bc242e2bb8341afa659ed166e08050",
+    "testWORD_3imgs.docx": "292ca6fa41d32b462e66061e89adb19423721975",
+    "testPDF_acroform3.pdf": "dcf6588cb5e41701b168606ea6bfbadecdcd3bc9",
+    "testWORD_missing_ooxml_bean1.docx": "c3058f2513fecc0a6d76d3ecf55676f236b085ff",
+    "testPDFTwoTextBoxes.pdf": "4adf324ce030076b1755fdb3a6cce676ee325ae4",
+    "testRTFUnicodeGothic.rtf": "f9932470ff686b0c217ea94ed5d4f2fd85f7998e",
+    "headers.mbox": "75ec25789fe870b6d25365e4ea73d731fc274847",
+    "testPPT_embeded.ppt": "",
+    "testXML3.xml": "804d4812408eb324ae8483d2140b648ec871dd2a",
+    "testOptionalHyphen.doc": "10f9ca38cc2985e94967aa2c454bfe40aff76976",
+    "testComment.doc": "66e57653d5d08478556ca640408b172b65855cc7",
+    "testEXCEL_headers_footers.xls": "18977c66fc8bcb8c44de3063b69b65a3de9c3f25",
+    "testWORD_embedded_rtf.doc": "cc2d289acfe3d1068a2649b7fa0c06c50bb6ceda",
+    "testEXCEL_custom_props.xlsx": "6b72ae08362a204b37dbba0a30b4134ae3e7918f",
+    "testOptionalHyphen.docx": "5b8ffc0df1691a8fed7d63aa9b256e9e02e36d71",
+    "testPPT_various.pptx": "d149de9af8071141a6ba6e2cd4ef5f6d9431a826",
+    "testWORD_closingSmartQInHyperLink.doc": "9859f378c603b70bf0d44a281169ae5b16a21878",
+    "test_embedded_zip.pptx": "d19406edcec09440d066877c451ceba60abc3483",
+    "testRTFUmlautSpaces.rtf": "155b39879c5b5fbad22fd650be37ae7f91489eb2",
+    "protectedFile.xlsx": "ee08eeaf05c35c960243f831c3a974d9ee07aa28",
+    "Doc1_ole.doc": "fb63220506ab666f1fe87b0608e1447fd4fd3489",
+    "testEXCEL_embeded.xlsx": "",
+    "EmbeddedDocument.docx": "",
+    "testODFwithOOo3.odt": "3815d6fb7f5829db882ea8ebd664f252711e6e60",
+    "testPagesHeadersFootersRomanUpper.pages": "85b3cd545ba6c33e5d44b844a6afea8cb6eaec0b",
+    "testPPT_comment.ppt": "88fd667fd0292785395a8d0d229304aa91110556",
+    "testPPT_2imgs.pptx": "66eda11ad472918153100dad8ee5be0f1f8e2e04",
+    "testPagesHeadersFootersAlphaUpper.pages": "56bef0d1eaedfd7599aae29031d2eeb0e3fe4688",
+    "testWORD_text_box.docx": "e01f7b05c6aac3449b9a699c3e4d2e62ff3368a3",
+    "testWORD_missing_text.docx": "3814332884a090b6d1020bff58d0531486710c45",
+    "testComment.pdf": "60e181061a00454c2e622bd37a9878234c13231d",
+    "testPDF_no_extract_no_accessibility_owner_empty.pdf": "6eb693dac68fece3bf3cd1aa9880ea9b23fc927c",
+    "test_embedded_package.rtf": "cd90adb3f777e68aa0288fd23e8f4fbce260a763",
+    "testPDF_bom.pdf": "6eb693dac68fece3bf3cd1aa9880ea9b23fc927c",
+    "testOptionalHyphen.ppt": "7e016e42860bd408054bb8653fef39b2756119d9",
+    "testHTML_utf8.html": "3ba828044754772e4c9df5f9a2213beaa75842ef",
+    "testPPT_comment.pptx": "25fab588194dabd5902fd2ef880ee9542d036776",
+    "testRTFWithCurlyBraces.rtf": "019cab63b73ff89d094823cf50c0a721bec08ee2",
+    "testFooter.ods": "846e1d0415b23fa27631b536b0cf566abbf8fcc1",
+    "testPPT.ppt": "933ee556884b1d9e28b801daa0d77bbaa4f4be62",
+    "testEXCEL-formats.xls": "",
+    "testPPT_masterFooter.pptx": "29bb97006b3608b7db6ff72b94d20157878d94dd",
+    "testWORD_header_hyperlink.doc": "914bbec0730c54948ad307ea3e375ef0c100abf1",
+    "testRTFHyperlink.rtf": "2b2ffb1997aa495fbab1af490d134051de168c97",
+    "testExtraSpaces.pdf": "b5575400309b01c1050a927d8d1ecf8761062abc",
+    "testRTFWindowsCodepage1250.rtf": "7ba418843f401634f97d21c844c2c4093b7194fb",
+    "testRTFTableCellSeparation2.rtf": "62782ca40ff0ed6c3ba90f8055ee724b44af203f",
+    "testPagesHeadersFootersRomanLower.pages": "2410fc803907001eb39c201ad4184b243e271c6d",
+    "headerPic.docx": "c704bb648feac7975dff1024a5f762325be7cbc2",
+    "testHTMLNoisyMetaEncoding_4.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a",
+    "testRTFBoldItalic.rtf": "0475d224078682cf3f9f3f4cbc14a63456c5a0d8",
+    "test-outlook.msg": "1f202fc11a873e305d5b4d4607409f3f734065ec",
+    "testRTFVarious.rtf": "bf6ea9cf57886e680c5e6743a66a12b950a09083",
+    "testXHTML.html": "c6da900f81c1c550518e65d579d3dd62dd7c5c0c",
+    "EmbeddedPDF.docx": "454476bdf4a968189a6f53e75c146382bf58a434",
+    "testXML.xml": "e1615e9b31be58f7af9ad963e5a112efa5cdaffa",
+    "testWORD_no_format.docx": "9a3f5d8a4c8c0f077cc615bcfc554dc87d5926aa",
+    "testPPT_masterText.ppt": "f5ff5e2d45ccb180cf371ed99b7dfeb2a93539b3",
+    "testPDF_PDFEncodedStringInXMP.pdf": "78fd59d394f72d28a9908739fa562099978dafa1",
+    "testPPT_custom_props.pptx": "72152d28afbc23a50cc71fa37d1dce9ef03ca72d",
+    "testRTFListOverride.rtf": "f8c61d8a66afdaa07f3740e859497818bfc2ca01",
+    "testEXCEL_1img.xls": "",
+    "testWORD_1img.doc": "0826d299a7770e93603f5667d89dccb7b74d904c",
+    "testNPEOpenDocument.odt": "4210b973c80084c58463ec637fa43e911f77d6fe",
+    "testRTFWord2010CzechCharacters.rtf": "9443011aac32434240ab8dbff360c970fc1c7074",
+    "testPDF_Version.8.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testPPT.ppsx": "71333ef84f7825d8ad6aba2ba993d04b4bab41c6",
+    "testPPT_autodate.pptx": "50467dbb37d1c74b8b37fe93eddf6f9e87d21bf3",
+    "testWordArt.pptx": "3566bbee790704b3654fe78319957f9e0cddb6d9",
+    "NullHeader.docx": "18430c968ba29173b52610efdaa723424b3c4d79",
+    "testRTFWordPadCzechCharacters.rtf": "5dbb58452a3507c384008662f8fce90063f12189",
+    "resume.html": "fbfb9d8264f6eebd79847fe7a7f1b81edd4a027d",
+    "testPagesLayout.pages": "5db1ab91c93e6183d0af8513f62c7b87964704af",
+    "testOptionalHyphen.pptx": "c2977eefe7d2cad8c671f550d7883185ec65591b",
+    "testWORD_numbered_list.docx": "07194c58165993468e66bc4eba4f5bd89d5bee09",
+    "testEXCEL_1img.xlsx": "",
+    "testPDFTripleLangTitle.pdf": "6eb693dac68fece3bf3cd1aa9880ea9b23fc927c",
+    "protect.xlsx": "ee08eeaf05c35c960243f831c3a974d9ee07aa28",
+    "testWORD_bold_character_runs2.docx": "f10e562d8825ec2e17e0d9f58646f8084a658cfa",
+    "testXLSX_Thumbnail.xlsx": "020bf155ae157661c11727c54e6694cf9cd2c0d3",
+    "testWORD_embedded_pdf.docx": "d8adb797aaaac92afd8dd9b499bd197347f15688",
+    "testOptionalHyphen.rtf": "2f77b61bab5b4502b4ddd5018b454be157091d07",
+    "testEXCEL-charts.xls": "",
+    "testWORD_override_list_numbering.doc": "60e47a3e71ba08af20af96131d61740a1f0bafa3",
+    "testPDF_twoAuthors.pdf": "c5f0296cc21f9ae99ceb649b561c55f99d7d9452",
+    "testPDF_Version.10.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testHTMLNoisyMetaEncoding_2.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a",
+    "testFooter.odt": "cd5d0fcbcf48d6f005d087c47d00e84f39bcc321",
+    "testPPT.pptm": "71333ef84f7825d8ad6aba2ba993d04b4bab41c6",
+    "testPPT_various.ppt": "399e27a9893284f106dc44f15b5e636454db681e",
+    "testRTFListMicrosoftWord.rtf": "0303eb3e2f30530621a7a407847b759a3b21467e",
+    "testWORD_bold_character_runs2.doc": "f10e562d8825ec2e17e0d9f58646f8084a658cfa",
+    "boilerplate-whitespace.html": "a9372bc75d7d84cbcbb0bce68fcaed73ad8ef52c",
+    "testEXCEL_95.xls": "20d9b9b0f3aecd28607516b4b837c8bab3524b6c",
+    "testPPT_embedded_two_slides.pptx": "",
+    "testPDF_bookmarks.pdf": "5fc486c443511452db4f1aa6530714c6aa49c831",
+    "test_recursive_embedded.docx": "afc32b07ce07ad273e5b3d1a43390a9d2b6dd0a9",
+    "testEXCEL-formats.xlsx": "",
+    "testPPT_masterText2.pptx": "2b01eab5d0349e3cfe791b28c70c2dbf4efc884d",
+    "test.doc": "774be3106edbb6d80be36dbb548d62401dcfa0fe",
+    "test_recursive_embedded_npe.docx": "afc32b07ce07ad273e5b3d1a43390a9d2b6dd0a9",
+    "testPPT_embedded2.ppt": "80e106b3fc68107e7f9579cff04e3b15bdfc557a",
+    "testWORD_custom_props.docx": "e7a737a5237a6aa9c6b3fc677eb8fa65c30d6dfe",
+    "testPDF_Version.4.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testBinControlWord.rtf": "ef858fbb7584ea7f92ffed8d0a08c1cc35ffee07",
+    "testWORD_null_style.docx": "0be9dcfb83423c78a06af514ec21e4e7770ec48e",
+    "test-outlook2003.msg": "bb3c35eb7e95d657d7977c1d3d52862734f9f329",
+    "testPDFVarious.pdf": "c66bbbacb10dd27430f7d0bed9518e75793cedae",
+    "testHTMLNoisyMetaEncoding_3.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a",
+    "testRTFCorruptListOverride.rtf": "116a782d02a7f25010a15cbbb189bf98e6b89855",
+    "testEXCEL_custom_props.xls": "b5584d9b13ab1566ce539238dc75e7eb3449ba7f",
+    "testPDF_Version.7.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testPDFEmbeddingAndEmbedded.docx": "e7b648adb15cd16cdd84437c2b9524a8eeb213e4",
+    "testHTMLNoisyMetaEncoding_1.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a",
+    "testWORD_3imgs.doc": "818aa8c6c44dd78c49100c3c38e95abdf3812981",
+    "testRTFEmbeddedLink.rtf": "2720ffb5ff3a6bbb2c5c1cb43fb4922362ed788a",
+    "testKeynote.key": "11387b59fc6339bb73653fcbb26d387521b98ec9",
+    "testPDF.pdf": "5a377554685367764eaf73d093408ace323fcec7",
+    "protectedSheets.xlsx": "",
+    "testWORD.doc": "cdd41377e699287cbbe17fbb1498cfe5814dde23",
+    "testComment.xlsx": "d4be580bb97c1c90be379281179c7932b37a18c0",
+    "testPDFPackage.pdf": "75d6fa216b4e2880a65ced55d17ca2b599d2606c",
+    "testWORD_embeded.doc": "",
+    "testHTML.html": "6548b16c5ea33e907577615ce60ca4876a3936ef",
+    "testEXCEL_5.xls": "a174f098333c659d331317641d4d1d9d83055288",
+    "pictures.ppt": "95bbfdbf2f60f74371285c337d3445d0acd59a9b",
+    "testPPT_masterText2.ppt": "f5ff5e2d45ccb180cf371ed99b7dfeb2a93539b3",
+    "testPDF-custommetadata.pdf": "a84b914655db55574e6002b6f37209ecd4c3d462",
+    "testWORD_embeded.docx": "",
+    "testStyles.odt": "c25dd05633e3aab7132d2f5608126e2b4b03848f",
+    "testPDF_multiFormatEmbFiles.pdf": "2103b2c30b44d5bb3aa790ab04a6741a10ea235a",
+    "testXML2.xml": "a8c85a327716fad93faa4eb0f993057597d6f471",
+    "testPagesComments.pages": "cbb45131cf45b9c454e754a07af3ae927b1a69cc",
+    "testEXCEL_4.xls": "8d5e6156222151faaccb079d46ddb5393dd25771",
+    "testWORD_no_format.doc": "88feaf03fe58ee5cc667916c6a54cbd5d605cc1c",
+    "testPages.pages": "288e6db2f39604e372a2095257509c78dba22cbb",
+    "footnotes.docx": "33b01b73a12f9e14efbcc340890b11ee332dca8e",
+    "testWORD_bold_character_runs.doc": "f10e562d8825ec2e17e0d9f58646f8084a658cfa",
+    "testWORD_custom_props.doc": "e7a737a5237a6aa9c6b3fc677eb8fa65c30d6dfe",
+    "testPDF_Version.11.x.PDFA-1b.pdf": "71853c6197a6a7f222db0f1978c7cb232b87c5ee",
+    "testAnnotations.pdf": "5f599e7916198540e1b52c3e472a525f50fd45f6",
+    "tika434.html": "7d74122631f52f003a48018cc376026ccd8d984e",
+    "testPagesHeadersFootersAlphaLower.pages": "fc1d766908134ff4689fa63fa3e91c3e9b08d975",
+    "testRTFRegularImages.rtf": "756b1db45cb05357ceaf9c8efcf0b76e3913e190",
+    "testRTFUmlautSpaces2.rtf": "1fcd029357062241d74d789e93477c101ff24e3f",
+    "testWORD_numbered_list.doc": "e06656dd9b79ac970f3cd065fa8b630a4981556f",
+    "testPPT_autodate.ppt": "05b93967ea0248ad263b2f24586e125df353fd3d",
+    "testBulletPoints.key": "92242d67c3dbc1b22aac3f98e47061d09e7719f9",
+    "testMasterSlideTable.key": "1d61e2fa3c3f3615500c7f72f62971391b9e9a2f",
+    "testWORD_various.doc": "8cbdf1a4e0d78471eb90403612c4e92866acf0cb",
+    "testEXCEL_textbox.xlsx": "1e81121e91e58a74d838e414ae0fc0055a4b4100",
+    "big-preamble.html": "a9d759b46b6c6c1857d0d89c3a75ee2f3ace70c9",
+    "testWORD.docx": "f72140bef19475e950e56084d1ab1cb926697b19",
+    "testComment.rtf": "f6351d0f1f20c4ee0fff70adca6abbc6e638610e",
+    "testRTFUnicodeUCNControlWordCharacterDoubling.rtf": "3e6f2f38682e38ffc96a476ca51bec2291a27fa7",
+    "testPDF_Version.5.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testPPTX_Thumbnail.pptx": "6aa019154289317c7b7832fe46556e6d61cd0a9f",
+    "testRTFTableCellSeparation.rtf": "5647290a3197c1855fad10201dc7be60ea7b0e42",
+    "testRTFControls.rtf": "aee6afb80e8b09cf49f056020c037f70c2757e49",
+    "testEXCEL.xls": "",
+    "testRTFJapanese.rtf": "08976f9a7d6d3a155cad84d7fa23295cb972a17a",
+    "testPageNumber.pdf": "96b03d2cc6782eba653af28228045964e68422b5",
+    "testOptionalHyphen.pdf": "12edd450ea76ea4e79f80ebd3442999ec2180dbc",
+    "testPDFFileEmbInAnnotation.pdf": "97a6e5781bbaa6aea040546d797c4916f9d90c86",
+    "testFontAfterBufferedText.rtf": "d1c8757b3ed91f2d7795234405c43005868affa3",
+    "testPPT_masterFooter.ppt": "8c9104385820c2631ddda20814231808fac03d4d",
+    "testWORD_various.docx": "189df989e80afb09281901aefc458c6630a8530b",
+    "testComment.ppt": "21842dd9cb8a7d4af0f102543c192861c9789705",
+    "testPopupAnnotation.pdf": "1717b1d16c0a4b9ff5790cac90fc8e0fba170a35",
+    "testWORD_bold_character_runs.docx": "f10e562d8825ec2e17e0d9f58646f8084a658cfa",
+    "testOverlappingText.pdf": "726da7d6c184512ed8d44af2a5085d65523c4572",
+    "testRTF.rtf": "91e830ceba556741116c9e83b0c69a0d6c5c9304",
+    "testRTFIgnoredControlWord.rtf": "1eb6a2f2fd32b1bb4227c0c02a35cb6027d9ec8c",
+    "testComment.xls": "4de962f16452159ce302fc4a412b06a06cf9a0f6",
+    "testPPT.ppsm": "71333ef84f7825d8ad6aba2ba993d04b4bab41c6",
+    "boilerplate.html": "b3558f02c3179e4aeeb6057594d87bda79964e7b",
+    "testEXCEL_embeded.xls": "",
+    "testEXCEL.xlsx": "",
+    "testPPT_2imgs.ppt": "9a68072ffcf171389e78cf8bc018c4b568a6202d",
+    "testComment.pptx": "6ae6052f469b8f901fd4fd8bc70f8e267255a58e",
+    "testPDF_Version.6.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testPPT.pptx": "71333ef84f7825d8ad6aba2ba993d04b4bab41c6",
+    "testPPT_custom_props.ppt": "edf196acc12701accc7be5dfe63e053436db45e6",
+    "testPPT_embeded.pptx": "",
+    "testRTFListLibreOffice.rtf": "4c38d9e2f0a8c9a4c2cc8d2a52db9591ab759abe",
+    "testPDF_Version.9.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2",
+    "testRTFHexEscapeInsideWord.rtf": "6cffda07e774c55b5465d8134a0bdcb8c30f3386",
+    "testRTFNewlines.rtf": "2375ca14e2b0d8f7ff6bbda5191544b3ee7c09fb",
+    "testRTF-ms932.rtf": "5f9db1b83bf8e9c4c6abb065adaeb151307d33f2",
+    "test_TIKA-1251.doc": "5a9394c34274964055fdd9272b4f7dc314b99ecf",
+    "test_list_override.rtf": "9fe8b4a36c5222fe7ed2e9b54e2330aec8fa9423"
+}

From f30e0e034ea004063c5d6470310ff237df6548ab Mon Sep 17 00:00:00 2001
From: gaobinlong <gbinlong@amazon.com>
Date: Fri, 17 May 2024 02:33:36 +0800
Subject: [PATCH 40/52] Fix
 org.opensearch.rest.action.admin.cluster.RestNodesStatsActionTests.testIndexMetricsRequestWithoutIndicesMetric
 is flaky (#13658)

* Fix org.opensearch.rest.action.admin.cluster.RestNodesStatsActionTests.testIndexMetricsRequestWithoutIndicesMetric is flaky

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

* Change test method name

Signed-off-by: Gao Binlong <gbinlong@amazon.com>

---------

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
---
 .../cluster/RestNodesStatsActionTests.java      | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
index 6883eccbf7427..818a2fa18d751 100644
--- a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
+++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
@@ -130,10 +130,12 @@ public void testUnrecognizedIndexMetricDidYouMean() {
         );
     }
 
-    public void testIndexMetricsRequestWithoutIndicesMetric() throws IOException {
+    public void testIndexMetricsRequestWithoutIndicesAndCachesMetrics() throws IOException {
         final HashMap<String, String> params = new HashMap<>();
         final Set<String> metrics = new HashSet<>(RestNodesStatsAction.METRICS.keySet());
         metrics.remove("indices");
+        // caches stats is handled separately
+        metrics.remove("caches");
         params.put("metric", randomSubsetOf(1, metrics).get(0));
         final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0);
         params.put("index_metric", indexMetric);
@@ -150,6 +152,19 @@ public void testIndexMetricsRequestWithoutIndicesMetric() throws IOException {
         );
     }
 
+    public void testCacheStatsRequestWithInvalidCacheType() throws IOException {
+        final HashMap<String, String> params = new HashMap<>();
+        params.put("metric", "caches");
+        final String cacheType = randomAlphaOfLength(64);
+        params.put("index_metric", cacheType);
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> action.prepareRequest(request, mock(NodeClient.class))
+        );
+        assertThat(e, hasToString(containsString("request [/_nodes/stats] contains unrecognized cache type: [" + cacheType + "]")));
+    }
+
     public void testIndexMetricsRequestOnAllRequest() throws IOException {
         final HashMap<String, String> params = new HashMap<>();
         params.put("metric", "_all");

From 4700be3e72e9193d438718120d1228222785786c Mon Sep 17 00:00:00 2001
From: Rishabh Singh <sngri@amazon.com>
Date: Thu, 16 May 2024 16:08:36 -0700
Subject: [PATCH 41/52] Add more identifiers to gradle-check job webhook
 payload (#13714)

---
 .github/workflows/gradle-check.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
index 1f5c187c28e7d..6d8fe02c7c9c3 100644
--- a/.github/workflows/gradle-check.yml
+++ b/.github/workflows/gradle-check.yml
@@ -30,11 +30,15 @@ jobs:
       - name: Setup environment variables (PR)
         if: github.event_name == 'pull_request_target'
         run: |
+          echo "event_name=pull_request_target" >> $GITHUB_ENV
+          echo "branch_name=$(jq --raw-output .pull_request.base.ref $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_owner=$(jq --raw-output .pull_request.user.login $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_or_commit_description=$(jq --ascii-output .pull_request.body $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
 
       - name: Setup environment variables (Push)
         if: github.event_name == 'push'
@@ -43,11 +47,14 @@ jobs:
           ref_id=$(git rev-parse HEAD)
           branch_name=$(git rev-parse --abbrev-ref HEAD)
           echo "branch_name=$branch_name" >> $GITHUB_ENV
+          echo "event_name=push" >> $GITHUB_ENV
           echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
           echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
+          echo "pr_owner=$(jq --raw-output '.commits[0].author.username' $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_number=Null" >> $GITHUB_ENV
+          echo "pr_or_commit_description=$(jq --ascii-output .head_commit.message $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
 
       - name: Checkout opensearch-build repo
         uses: actions/checkout@v4

From 6ba6f5962c6d90188b94e89a02c59c8d247fd910 Mon Sep 17 00:00:00 2001
From: Gagan Juneja <gagandeepjuneja@gmail.com>
Date: Fri, 17 May 2024 22:47:00 +0530
Subject: [PATCH 42/52] Adds support to inject telemetry instances to plugins
 (#13636)

* Adds support to inject telemetry instances to plugins

Signed-off-by: Gagan Juneja <gjjuneja@amazon.com>

* Adds test

Signed-off-by: Gagan Juneja <gjjuneja@amazon.com>

* incorporate pr comments

Signed-off-by: Gagan Juneja <gjjuneja@amazon.com>

---------

Signed-off-by: Gagan Juneja <gjjuneja@amazon.com>
Co-authored-by: Gagan Juneja <gjjuneja@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../main/java/org/opensearch/node/Node.java   | 38 ++++++++
 .../plugins/TelemetryAwarePlugin.java         | 80 ++++++++++++++++
 .../java/org/opensearch/node/NodeTests.java   | 94 +++++++++++++++++++
 4 files changed, 213 insertions(+)
 create mode 100644 server/src/main/java/org/opensearch/plugins/TelemetryAwarePlugin.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f868dd76039b..ba19e8d55dab3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
 - Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
+- Adds support to inject telemetry instances to plugins ([#13636](https://github.com/opensearch-project/OpenSearch/pull/13636))
 
 ### Deprecated
 
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 614f39166ea66..0b25126f6be82 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -207,6 +207,7 @@
 import org.opensearch.plugins.SearchPlugin;
 import org.opensearch.plugins.SecureSettingsFactory;
 import org.opensearch.plugins.SystemIndexPlugin;
+import org.opensearch.plugins.TelemetryAwarePlugin;
 import org.opensearch.plugins.TelemetryPlugin;
 import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
 import org.opensearch.ratelimitting.admissioncontrol.transport.AdmissionControlTransportInterceptor;
@@ -274,6 +275,7 @@
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -620,6 +622,18 @@ protected Node(
                 final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings());
                 if (telemetrySettings.isTracingFeatureEnabled() || telemetrySettings.isMetricsFeatureEnabled()) {
                     List<TelemetryPlugin> telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class);
+                    List<TelemetryPlugin> telemetryPluginsImplementingTelemetryAware = telemetryPlugins.stream()
+                        .filter(a -> TelemetryAwarePlugin.class.isAssignableFrom(a.getClass()))
+                        .collect(toList());
+                    if (telemetryPluginsImplementingTelemetryAware.isEmpty() == false) {
+                        throw new IllegalStateException(
+                            String.format(
+                                Locale.ROOT,
+                                "Telemetry plugins %s should not implement TelemetryAwarePlugin interface",
+                                telemetryPluginsImplementingTelemetryAware
+                            )
+                        );
+                    }
                     TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings);
                     if (telemetrySettings.isTracingFeatureEnabled()) {
                         tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext());
@@ -909,6 +923,30 @@ protected Node(
                 )
                 .collect(Collectors.toList());
 
+            Collection<Object> telemetryAwarePluginComponents = pluginsService.filterPlugins(TelemetryAwarePlugin.class)
+                .stream()
+                .flatMap(
+                    p -> p.createComponents(
+                        client,
+                        clusterService,
+                        threadPool,
+                        resourceWatcherService,
+                        scriptService,
+                        xContentRegistry,
+                        environment,
+                        nodeEnvironment,
+                        namedWriteableRegistry,
+                        clusterModule.getIndexNameExpressionResolver(),
+                        repositoriesServiceReference::get,
+                        tracer,
+                        metricsRegistry
+                    ).stream()
+                )
+                .collect(Collectors.toList());
+
+            // Add the telemetryAwarePlugin components to the existing pluginComponents collection.
+            pluginComponents.addAll(telemetryAwarePluginComponents);
+
             // register all standard SearchRequestOperationsCompositeListenerFactory to the SearchRequestOperationsCompositeListenerFactory
             final SearchRequestOperationsCompositeListenerFactory searchRequestOperationsCompositeListenerFactory =
                 new SearchRequestOperationsCompositeListenerFactory(
diff --git a/server/src/main/java/org/opensearch/plugins/TelemetryAwarePlugin.java b/server/src/main/java/org/opensearch/plugins/TelemetryAwarePlugin.java
new file mode 100644
index 0000000000000..42cab326f88bf
--- /dev/null
+++ b/server/src/main/java/org/opensearch/plugins/TelemetryAwarePlugin.java
@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugins;
+
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.lifecycle.LifecycleComponent;
+import org.opensearch.core.common.io.stream.NamedWriteable;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.env.Environment;
+import org.opensearch.env.NodeEnvironment;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.script.ScriptService;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.watcher.ResourceWatcherService;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.function.Supplier;
+
+/**
+ * Plugin that provides the telemetry registries to build component with telemetry and also provide a way to
+ * pass telemetry registries to the implementing plugins for adding instrumentation in the code.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface TelemetryAwarePlugin {
+
+    /**
+     * Returns components added by this plugin.
+     * <p>
+     * Any components returned that implement {@link LifecycleComponent} will have their lifecycle managed.
+     * Note: To aid in the migration away from guice, all objects returned as components will be bound in guice
+     * to themselves.
+     *
+     * @param client A client to make requests to the system
+     * @param clusterService A service to allow watching and updating cluster state
+     * @param threadPool A service to allow retrieving an executor to run an async action
+     * @param resourceWatcherService A service to watch for changes to node local files
+     * @param scriptService A service to allow running scripts on the local node
+     * @param xContentRegistry the registry for extensible xContent parsing
+     * @param environment the environment for path and setting configurations
+     * @param nodeEnvironment the node environment used coordinate access to the data paths
+     * @param namedWriteableRegistry the registry for {@link NamedWriteable} object parsing
+     * @param indexNameExpressionResolver A service that resolves expression to index and alias names
+     * @param repositoriesServiceSupplier A supplier for the service that manages snapshot repositories; will return null when this method
+     *                                   is called, but will return the repositories service once the node is initialized.
+     * @param tracer the tracer to add tracing instrumentation.
+     * @param metricsRegistry the registry for metrics instrumentation.
+     */
+    default Collection<Object> createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<RepositoriesService> repositoriesServiceSupplier,
+        Tracer tracer,
+        MetricsRegistry metricsRegistry
+    ) {
+        return Collections.emptyList();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/node/NodeTests.java b/server/src/test/java/org/opensearch/node/NodeTests.java
index d91dc696eb30b..f44cc352cd330 100644
--- a/server/src/test/java/org/opensearch/node/NodeTests.java
+++ b/server/src/test/java/org/opensearch/node/NodeTests.java
@@ -34,17 +34,23 @@
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.bootstrap.BootstrapCheck;
 import org.opensearch.bootstrap.BootstrapContext;
+import org.opensearch.client.Client;
 import org.opensearch.cluster.ClusterName;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.SetOnce;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.transport.BoundTransportAddress;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.index.IndexService;
@@ -56,22 +62,35 @@
 import org.opensearch.monitor.fs.FsProbe;
 import org.opensearch.plugins.CircuitBreakerPlugin;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.TelemetryAwarePlugin;
+import org.opensearch.plugins.TelemetryPlugin;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.script.ScriptService;
+import org.opensearch.telemetry.Telemetry;
+import org.opensearch.telemetry.TelemetrySettings;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.test.FeatureFlagSetter;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.MockHttpTransport;
 import org.opensearch.test.NodeRoles;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
@@ -404,6 +423,81 @@ public void testCreateWithFileCache() throws Exception {
         }
     }
 
+    public void testTelemetryAwarePlugins() throws IOException {
+        Settings.Builder settings = baseSettings();
+        List<Class<? extends Plugin>> plugins = basePlugins();
+        plugins.add(MockTelemetryAwarePlugin.class);
+        try (Node node = new MockNode(settings.build(), plugins)) {
+            MockTelemetryAwareComponent mockTelemetryAwareComponent = node.injector().getInstance(MockTelemetryAwareComponent.class);
+            assertNotNull(mockTelemetryAwareComponent.getTracer());
+            assertNotNull(mockTelemetryAwareComponent.getMetricsRegistry());
+            TelemetryAwarePlugin telemetryAwarePlugin = node.getPluginsService().filterPlugins(TelemetryAwarePlugin.class).get(0);
+            assertTrue(telemetryAwarePlugin instanceof MockTelemetryAwarePlugin);
+        }
+    }
+
+    public void testTelemetryPluginShouldNOTImplementTelemetryAwarePlugin() throws IOException {
+        Settings.Builder settings = baseSettings();
+        List<Class<? extends Plugin>> plugins = basePlugins();
+        plugins.add(MockTelemetryPlugin.class);
+        FeatureFlagSetter.set(FeatureFlags.TELEMETRY);
+        settings.put(TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING.getKey(), true);
+        assertThrows(IllegalStateException.class, () -> new MockNode(settings.build(), plugins));
+    }
+
+    private static class MockTelemetryAwareComponent {
+        private final Tracer tracer;
+        private final MetricsRegistry metricsRegistry;
+
+        public MockTelemetryAwareComponent(Tracer tracer, MetricsRegistry metricsRegistry) {
+            this.tracer = tracer;
+            this.metricsRegistry = metricsRegistry;
+        }
+
+        public Tracer getTracer() {
+            return tracer;
+        }
+
+        public MetricsRegistry getMetricsRegistry() {
+            return metricsRegistry;
+        }
+    }
+
+    public static class MockTelemetryAwarePlugin extends Plugin implements TelemetryAwarePlugin {
+        @Override
+        public Collection<Object> createComponents(
+            Client client,
+            ClusterService clusterService,
+            ThreadPool threadPool,
+            ResourceWatcherService resourceWatcherService,
+            ScriptService scriptService,
+            NamedXContentRegistry xContentRegistry,
+            Environment environment,
+            NodeEnvironment nodeEnvironment,
+            NamedWriteableRegistry namedWriteableRegistry,
+            IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<RepositoriesService> repositoriesServiceSupplier,
+            Tracer tracer,
+            MetricsRegistry metricsRegistry
+        ) {
+            return List.of(new MockTelemetryAwareComponent(tracer, metricsRegistry));
+        }
+
+    }
+
+    public static class MockTelemetryPlugin extends Plugin implements TelemetryPlugin, TelemetryAwarePlugin {
+
+        @Override
+        public Optional<Telemetry> getTelemetry(TelemetrySettings telemetrySettings) {
+            return Optional.empty();
+        }
+
+        @Override
+        public String getName() {
+            return null;
+        }
+    }
+
     public static class MockCircuitBreakerPlugin extends Plugin implements CircuitBreakerPlugin {
 
         private SetOnce<CircuitBreaker> myCircuitBreaker = new SetOnce<>();

From d26cd462c6b807078f3c871b89fdbd96122c016e Mon Sep 17 00:00:00 2001
From: Sandesh Kumar <sandeshkr419@gmail.com>
Date: Fri, 17 May 2024 13:46:38 -0700
Subject: [PATCH 43/52] TermsAggregatorTests flaky test fix (#13567)

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>
---
 .../bucket/terms/TermsAggregatorTests.java      | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index 6d105c27a692f..d97cfdf003600 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -353,33 +353,40 @@ private void testSimple(
                     newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE)
                 )
             ) {
+                List<Document> documents = new ArrayList<>();
                 Document document = new Document();
                 addFieldConsumer.apply(document, "string", "a");
                 addFieldConsumer.apply(document, "string", "b");
-                indexWriter.addDocument(document);
+                documents.add(document);
+
                 document = new Document();
                 addFieldConsumer.apply(document, "string", "");
                 addFieldConsumer.apply(document, "string", "c");
                 addFieldConsumer.apply(document, "string", "a");
-                indexWriter.addDocument(document);
+                documents.add(document);
+
                 document = new Document();
                 addFieldConsumer.apply(document, "string", "b");
                 addFieldConsumer.apply(document, "string", "d");
-                indexWriter.addDocument(document);
+                documents.add(document);
+
                 document = new Document();
                 addFieldConsumer.apply(document, "string", "");
                 if (includeDocCountField) {
                     // Adding _doc_count to one document
                     document.add(new NumericDocValuesField("_doc_count", 10));
                 }
-                indexWriter.addDocument(document);
+                documents.add(document);
 
                 if (includeDeletedDocumentsInSegment) {
                     document = new Document();
                     ADD_SORTED_SET_FIELD_INDEXED.apply(document, "string", "e");
-                    indexWriter.addDocument(document);
+                    documents.add(document);
+                    indexWriter.addDocuments(documents);
                     indexWriter.deleteDocuments(new Term("string", "e"));
                     assertEquals(5, indexWriter.getDocStats().maxDoc);  // deleted document still in segment
+                } else {
+                    indexWriter.addDocuments(documents);
                 }
 
                 try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {

From a254aa99e2221561c4de79a75d0de237eb759851 Mon Sep 17 00:00:00 2001
From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com>
Date: Mon, 20 May 2024 19:03:22 +0530
Subject: [PATCH 44/52] Adds latency metrics for ClusterState Appliers and
 Listeners (#12333)

* Adds latency metrics for ClusterState Appliers and Listeners

Signed-off-by: Harsh Garg <gkharsh@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../insights/QueryInsightsPluginTests.java    |  4 +-
 .../listener/QueryInsightsListenerTests.java  |  3 +-
 .../TransportTopQueriesActionTests.java       |  3 +-
 .../cluster/ClusterManagerMetrics.java        | 72 +++++++++++++++++++
 .../org/opensearch/cluster/ClusterModule.java |  6 +-
 .../routing/allocation/AllocationService.java | 28 ++++++--
 .../service/ClusterApplierService.java        | 29 +++++++-
 .../service/ClusterManagerService.java        | 10 ++-
 .../cluster/service/ClusterService.java       | 12 +++-
 .../cluster/service/MasterService.java        | 19 ++++-
 .../main/java/org/opensearch/node/Node.java   | 35 +++++----
 ...ActionIndicesThatCannotBeCreatedTests.java |  7 +-
 .../bulk/TransportBulkActionIngestTests.java  |  7 +-
 .../search/SearchRequestSlowLogTests.java     | 23 +++---
 .../cluster/ClusterModuleTests.java           | 26 +++++--
 ...rnalClusterInfoServiceSchedulingTests.java |  9 ++-
 .../cluster/coordination/NodeJoinTests.java   |  5 +-
 .../allocation/AllocationServiceTests.java    | 33 ++++++++-
 .../service/ClusterApplierServiceTests.java   | 69 +++++++++++++++++-
 .../cluster/service/ClusterServiceTests.java  |  5 +-
 .../cluster/service/MasterServiceTests.java   | 54 ++++++++++++--
 .../gateway/ClusterStateUpdatersTests.java    |  3 +-
 .../gateway/GatewayServiceTests.java          |  3 +-
 .../index/IndexingPressureServiceTests.java   |  3 +-
 ...exingPressureConcurrentExecutionTests.java |  3 +-
 ...ardIndexingPressureMemoryManagerTests.java |  4 +-
 .../ShardIndexingPressureSettingsTests.java   |  3 +-
 .../ShardIndexingPressureStoreTests.java      |  4 +-
 .../index/ShardIndexingPressureTests.java     |  3 +-
 .../RemoteSegmentTransferTrackerTests.java    |  3 +-
 .../RemoteStorePressureServiceTests.java      |  3 +-
 .../RemoteStorePressureSettingsTests.java     |  3 +-
 .../RemoteStoreStatsTrackerFactoryTests.java  | 19 ++++-
 .../RemoteStoreRefreshListenerTests.java      |  5 +-
 .../node/ResponseCollectorServiceTests.java   |  3 +-
 .../AdmissionControlServiceTests.java         |  3 +-
 .../AdmissionControlSettingsTests.java        |  3 +-
 .../CpuBasedAdmissionControllerTests.java     |  3 +-
 .../IoBasedAdmissionControllerTests.java      |  3 +-
 ...BasedAdmissionControllerSettingsTests.java |  3 +-
 ...BasedAdmissionControllerSettingsTests.java |  3 +-
 .../stats/AdmissionControlStatsTests.java     |  3 +-
 .../stats/AdmissionControllerStatsTests.java  |  3 +-
 .../snapshots/SnapshotResiliencyTests.java    |  9 ++-
 .../AbstractCoordinatorTestCase.java          | 10 ++-
 .../FakeThreadPoolClusterManagerService.java  |  5 +-
 .../opensearch/test/ClusterServiceUtils.java  | 26 ++++++-
 48 files changed, 495 insertions(+), 101 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba19e8d55dab3..5060409cec3db 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add latency metrics for instrumenting critical clusterManager code paths ([#12333](https://github.com/opensearch-project/OpenSearch/pull/12333))
 - Add support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423))
 - Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
 - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java
index 273b69e483e8c..2f353f2a53329 100644
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java
+++ b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java
@@ -21,6 +21,7 @@
 import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
 import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.rest.RestHandler;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.ExecutorBuilder;
 import org.opensearch.threadpool.ScalingExecutorBuilder;
@@ -50,8 +51,7 @@ public void setup() {
         clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE);
         clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE);
 
-        clusterService = new ClusterService(settings, clusterSettings, threadPool);
-
+        clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, threadPool);
     }
 
     public void testGetSettings() {
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java
index f340950017a5c..328ed0cd2ed15 100644
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java
+++ b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListenerTests.java
@@ -22,6 +22,7 @@
 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.opensearch.search.aggregations.support.ValueType;
 import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
@@ -57,7 +58,7 @@ public void setup() {
         clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED);
         clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE);
         clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE);
-        clusterService = new ClusterService(settings, clusterSettings, null);
+        clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         when(queryInsightsService.isCollectionEnabled(MetricType.LATENCY)).thenReturn(true);
         when(queryInsightsService.getTopQueriesService(MetricType.LATENCY)).thenReturn(topQueriesService);
     }
diff --git a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java
index a5f36b6e8cce0..d05cf7b6a636f 100644
--- a/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java
+++ b/plugins/query-insights/src/test/java/org/opensearch/plugin/insights/rules/transport/top_queries/TransportTopQueriesActionTests.java
@@ -17,6 +17,7 @@
 import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesResponse;
 import org.opensearch.plugin.insights.rules.model.MetricType;
 import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
@@ -33,7 +34,7 @@ public class TransportTopQueriesActionTests extends OpenSearchTestCase {
     private final Settings.Builder settingsBuilder = Settings.builder();
     private final Settings settings = settingsBuilder.build();
     private final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-    private final ClusterService clusterService = new ClusterService(settings, clusterSettings, threadPool);
+    private final ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, threadPool);
     private final TransportService transportService = mock(TransportService.class);
     private final QueryInsightsService topQueriesByLatencyService = mock(QueryInsightsService.class);
     private final ActionFilters actionFilters = mock(ActionFilters.class);
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java b/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
new file mode 100644
index 0000000000000..d48f82a388245
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster;
+
+import org.opensearch.telemetry.metrics.Histogram;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.tags.Tags;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Class containing metrics (counters/latency) specific to ClusterManager.
+ *
+ * @opensearch.internal
+ */
+public final class ClusterManagerMetrics {
+
+    private static final String LATENCY_METRIC_UNIT_MS = "ms";
+
+    public final Histogram clusterStateAppliersHistogram;
+    public final Histogram clusterStateListenersHistogram;
+    public final Histogram rerouteHistogram;
+    public final Histogram clusterStateComputeHistogram;
+    public final Histogram clusterStatePublishHistogram;
+
+    public ClusterManagerMetrics(MetricsRegistry metricsRegistry) {
+        clusterStateAppliersHistogram = metricsRegistry.createHistogram(
+            "cluster.state.appliers.latency",
+            "Histogram for tracking the latency of cluster state appliers",
+            LATENCY_METRIC_UNIT_MS
+        );
+        clusterStateListenersHistogram = metricsRegistry.createHistogram(
+            "cluster.state.listeners.latency",
+            "Histogram for tracking the latency of cluster state listeners",
+            LATENCY_METRIC_UNIT_MS
+        );
+        rerouteHistogram = metricsRegistry.createHistogram(
+            "allocation.reroute.latency",
+            "Histogram for recording latency of shard re-routing",
+            LATENCY_METRIC_UNIT_MS
+        );
+        clusterStateComputeHistogram = metricsRegistry.createHistogram(
+            "cluster.state.new.compute.latency",
+            "Histogram for recording time taken to compute new cluster state",
+            LATENCY_METRIC_UNIT_MS
+        );
+        clusterStatePublishHistogram = metricsRegistry.createHistogram(
+            "cluster.state.publish.success.latency",
+            "Histogram for recording time taken to publish a new cluster state",
+            LATENCY_METRIC_UNIT_MS
+        );
+    }
+
+    public void recordLatency(Histogram histogram, Double value) {
+        histogram.record(value);
+    }
+
+    public void recordLatency(Histogram histogram, Double value, Optional<Tags> tags) {
+        if (Objects.isNull(tags) || tags.isEmpty()) {
+            histogram.record(value);
+            return;
+        }
+        histogram.record(value, tags.get());
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
index aa9101090b6d5..f56c906db1002 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
@@ -146,7 +146,8 @@ public ClusterModule(
         List<ClusterPlugin> clusterPlugins,
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService,
-        ThreadContext threadContext
+        ThreadContext threadContext,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.clusterPlugins = clusterPlugins;
         this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins);
@@ -159,7 +160,8 @@ public ClusterModule(
             shardsAllocator,
             clusterInfoService,
             snapshotsInfoService,
-            settings
+            settings,
+            clusterManagerMetrics
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
index 71e562253bf58..3864e282a310b 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
@@ -37,6 +37,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.RestoreInProgress;
 import org.opensearch.cluster.health.ClusterHealthStatus;
@@ -56,10 +57,12 @@
 import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.opensearch.cluster.routing.allocation.decider.Decision;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.gateway.GatewayAllocator;
 import org.opensearch.gateway.PriorityComparator;
 import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 import org.opensearch.snapshots.SnapshotsInfoService;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -96,6 +99,7 @@ public class AllocationService {
     private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
     private SnapshotsInfoService snapshotsInfoService;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
     // only for tests that use the GatewayAllocator as the unique ExistingShardsAllocator
     public AllocationService(
@@ -105,7 +109,13 @@ public AllocationService(
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService
     ) {
-        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService);
+        this(
+            allocationDeciders,
+            shardsAllocator,
+            clusterInfoService,
+            snapshotsInfoService,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        );
         setExistingShardsAllocators(Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, gatewayAllocator));
     }
 
@@ -113,9 +123,10 @@ public AllocationService(
         AllocationDeciders allocationDeciders,
         ShardsAllocator shardsAllocator,
         ClusterInfoService clusterInfoService,
-        SnapshotsInfoService snapshotsInfoService
+        SnapshotsInfoService snapshotsInfoService,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
-        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService, Settings.EMPTY);
+        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService, Settings.EMPTY, clusterManagerMetrics);
     }
 
     public AllocationService(
@@ -123,14 +134,15 @@ public AllocationService(
         ShardsAllocator shardsAllocator,
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService,
-        Settings settings
-
+        Settings settings,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.allocationDeciders = allocationDeciders;
         this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
         this.snapshotsInfoService = snapshotsInfoService;
         this.settings = settings;
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     /**
@@ -550,11 +562,15 @@ private void reroute(RoutingAllocation allocation) {
         assert AutoExpandReplicas.getAutoExpandReplicaChanges(allocation.metadata(), allocation).isEmpty()
             : "auto-expand replicas out of sync with number of nodes in the cluster";
         assert assertInitialized();
-
+        long rerouteStartTimeNS = System.nanoTime();
         removeDelayMarkers(allocation);
 
         allocateExistingUnassignedShards(allocation);  // try to allocate existing shard copies first
         shardsAllocator.allocate(allocation);
+        clusterManagerMetrics.recordLatency(
+            clusterManagerMetrics.rerouteHistogram,
+            (double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - rerouteStartTimeNS))
+        );
         assert RoutingNodes.assertShardStats(allocation.routingNodes());
     }
 
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
index a55721fb13cdc..2ac95178d2ff9 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateApplier;
 import org.opensearch.cluster.ClusterStateListener;
@@ -61,6 +62,7 @@
 import org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.telemetry.metrics.tags.Tags;
 import org.opensearch.threadpool.Scheduler;
 import org.opensearch.threadpool.ThreadPool;
 
@@ -68,6 +70,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -120,8 +123,15 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements
     private final String nodeName;
 
     private NodeConnectionsService nodeConnectionsService;
-
-    public ClusterApplierService(String nodeName, Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+    private final ClusterManagerMetrics clusterManagerMetrics;
+
+    public ClusterApplierService(
+        String nodeName,
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ThreadPool threadPool,
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
         this.clusterSettings = clusterSettings;
         this.threadPool = threadPool;
         this.state = new AtomicReference<>();
@@ -132,6 +142,7 @@ public ClusterApplierService(String nodeName, Settings settings, ClusterSettings
             CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
             this::setSlowTaskLoggingThreshold
         );
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
@@ -597,7 +608,7 @@ private void callClusterStateAppliers(ClusterChangedEvent clusterChangedEvent, S
         callClusterStateAppliers(clusterChangedEvent, stopWatch, lowPriorityStateAppliers);
     }
 
-    private static void callClusterStateAppliers(
+    private void callClusterStateAppliers(
         ClusterChangedEvent clusterChangedEvent,
         StopWatch stopWatch,
         Collection<ClusterStateApplier> clusterStateAppliers
@@ -605,7 +616,13 @@ private static void callClusterStateAppliers(
         for (ClusterStateApplier applier : clusterStateAppliers) {
             logger.trace("calling [{}] with change to version [{}]", applier, clusterChangedEvent.state().version());
             try (TimingHandle ignored = stopWatch.timing("running applier [" + applier + "]")) {
+                long applierStartTimeNS = System.nanoTime();
                 applier.applyClusterState(clusterChangedEvent);
+                clusterManagerMetrics.recordLatency(
+                    clusterManagerMetrics.clusterStateAppliersHistogram,
+                    (double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - applierStartTimeNS)),
+                    Optional.of(Tags.create().addTag("Operation", applier.getClass().getSimpleName()))
+                );
             }
         }
     }
@@ -624,7 +641,13 @@ private void callClusterStateListener(
             try {
                 logger.trace("calling [{}] with change to version [{}]", listener, clusterChangedEvent.state().version());
                 try (TimingHandle ignored = stopWatch.timing("notifying listener [" + listener + "]")) {
+                    long listenerStartTimeNS = System.nanoTime();
                     listener.clusterChanged(clusterChangedEvent);
+                    clusterManagerMetrics.recordLatency(
+                        clusterManagerMetrics.clusterStateListenersHistogram,
+                        (double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - listenerStartTimeNS)),
+                        Optional.of(Tags.create().addTag("Operation", listener.getClass().getSimpleName()))
+                    );
                 }
             } catch (Exception ex) {
                 logger.warn("failed to notify ClusterStateListener", ex);
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java
index e9224596e048d..eaedb36a59f1e 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.cluster.service;
 
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
@@ -20,7 +21,12 @@
  */
 @PublicApi(since = "2.2.0")
 public class ClusterManagerService extends MasterService {
-    public ClusterManagerService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
-        super(settings, clusterSettings, threadPool);
+    public ClusterManagerService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ThreadPool threadPool,
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
+        super(settings, clusterSettings, threadPool, clusterManagerMetrics);
     }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
index aa7766979e851..fa61375e85c25 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.cluster.service;
 
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateApplier;
@@ -91,12 +92,17 @@ public class ClusterService extends AbstractLifecycleComponent {
 
     private IndexingPressureService indexingPressureService;
 
-    public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+    public ClusterService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ThreadPool threadPool,
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
         this(
             settings,
             clusterSettings,
-            new ClusterManagerService(settings, clusterSettings, threadPool),
-            new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool)
+            new ClusterManagerService(settings, clusterSettings, threadPool, clusterManagerMetrics),
+            new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool, clusterManagerMetrics)
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
index af3e4f8437c43..6436dcfe33003 100644
--- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.AckedClusterStateTaskListener;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterState.Builder;
 import org.opensearch.cluster.ClusterStateTaskConfig;
@@ -70,6 +71,7 @@
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.discovery.Discovery;
 import org.opensearch.node.Node;
+import org.opensearch.telemetry.metrics.tags.Tags;
 import org.opensearch.threadpool.Scheduler;
 import org.opensearch.threadpool.ThreadPool;
 
@@ -79,6 +81,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -135,8 +138,14 @@ public class MasterService extends AbstractLifecycleComponent {
     protected final ClusterManagerTaskThrottler clusterManagerTaskThrottler;
     private final ClusterManagerThrottlingStats throttlingStats;
     private final ClusterStateStats stateStats;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
-    public MasterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+    public MasterService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ThreadPool threadPool,
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
         this.nodeName = Objects.requireNonNull(Node.NODE_NAME_SETTING.get(settings));
 
         this.slowTaskLoggingThreshold = CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(settings);
@@ -154,6 +163,7 @@ public MasterService(Settings settings, ClusterSettings clusterSettings, ThreadP
         );
         this.stateStats = new ClusterStateStats();
         this.threadPool = threadPool;
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
@@ -303,6 +313,12 @@ private void runTasks(TaskInputs taskInputs) {
         final TimeValue computationTime = getTimeSince(computationStartTime);
         logExecutionTime(computationTime, "compute cluster state update", summary);
 
+        clusterManagerMetrics.recordLatency(
+            clusterManagerMetrics.clusterStateComputeHistogram,
+            (double) computationTime.getMillis(),
+            Optional.of(Tags.create().addTag("Operation", taskInputs.executor.getClass().getSimpleName()))
+        );
+
         if (taskOutputs.clusterStateUnchanged()) {
             final long notificationStartTime = threadPool.preciseRelativeTimeInNanos();
             taskOutputs.notifySuccessfulTasksOnUnchangedClusterState();
@@ -361,6 +377,7 @@ protected boolean blockingAllowed() {
             final long durationMillis = getTimeSince(startTimeNanos).millis();
             stateStats.stateUpdateTook(durationMillis);
             stateStats.stateUpdated();
+            clusterManagerMetrics.recordLatency(clusterManagerMetrics.clusterStatePublishHistogram, (double) durationMillis);
         } catch (Exception e) {
             stateStats.stateUpdateFailed();
             onPublicationFailed(clusterChangedEvent, taskOutputs, startTimeNanos, e);
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 0b25126f6be82..c0ceba23ccbf6 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -59,6 +59,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
@@ -605,21 +606,10 @@ protected Node(
                 getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class))
             );
 
-            List<ClusterPlugin> clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class);
-            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
-            clusterService.addStateApplier(scriptService);
-            resourcesToClose.add(clusterService);
-            final Set<Setting<?>> consistentSettings = settingsModule.getConsistentSettings();
-            if (consistentSettings.isEmpty() == false) {
-                clusterService.addLocalNodeMasterListener(
-                    new ConsistentSettingsService(settings, clusterService, consistentSettings).newHashPublisher()
-                );
-            }
-
             TracerFactory tracerFactory;
             MetricsRegistryFactory metricsRegistryFactory;
             if (FeatureFlags.isEnabled(TELEMETRY)) {
-                final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings());
+                final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, settingsModule.getClusterSettings());
                 if (telemetrySettings.isTracingFeatureEnabled() || telemetrySettings.isMetricsFeatureEnabled()) {
                     List<TelemetryPlugin> telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class);
                     List<TelemetryPlugin> telemetryPluginsImplementingTelemetryAware = telemetryPlugins.stream()
@@ -659,6 +649,24 @@ protected Node(
             resourcesToClose.add(tracer::close);
             resourcesToClose.add(metricsRegistry::close);
 
+            final ClusterManagerMetrics clusterManagerMetrics = new ClusterManagerMetrics(metricsRegistry);
+
+            List<ClusterPlugin> clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class);
+            final ClusterService clusterService = new ClusterService(
+                settings,
+                settingsModule.getClusterSettings(),
+                threadPool,
+                clusterManagerMetrics
+            );
+            clusterService.addStateApplier(scriptService);
+            resourcesToClose.add(clusterService);
+            final Set<Setting<?>> consistentSettings = settingsModule.getConsistentSettings();
+            if (consistentSettings.isEmpty() == false) {
+                clusterService.addLocalNodeMasterListener(
+                    new ConsistentSettingsService(settings, clusterService, consistentSettings).newHashPublisher()
+                );
+            }
+
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
             final UsageService usageService = new UsageService();
 
@@ -686,7 +694,8 @@ protected Node(
                 clusterPlugins,
                 clusterInfoService,
                 snapshotsInfoService,
-                threadPool.getThreadContext()
+                threadPool.getThreadContext(),
+                clusterManagerMetrics
             );
             modules.add(clusterModule);
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
index cf7080ab2fc06..ff9e41ee7c784 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -55,6 +55,7 @@
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.tasks.Task;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.ThreadPool;
@@ -153,7 +154,11 @@ private void indicesThatCannotBeCreatedTestCase(
             null,
             new IndexingPressureService(
                 Settings.EMPTY,
-                new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null)
+                ClusterServiceUtils.createClusterService(
+                    Settings.EMPTY,
+                    new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                    null
+                )
             ),
             null,
             new SystemIndices(emptyMap()),
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
index da9156ccdb71a..a94a5d60b3f5a 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
@@ -71,6 +71,7 @@
 import org.opensearch.ingest.IngestService;
 import org.opensearch.tasks.Task;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.ThreadPool;
@@ -170,7 +171,11 @@ class TestTransportBulkAction extends TransportBulkAction {
                 ),
                 new IndexingPressureService(
                     SETTINGS,
-                    new ClusterService(SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null)
+                    ClusterServiceUtils.createClusterService(
+                        SETTINGS,
+                        new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                        null
+                    )
                 ),
                 null,
                 new SystemIndices(emptyMap()),
diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java
index f009988ffae17..91a2552ac3f04 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java
@@ -45,6 +45,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -90,7 +91,7 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() {
         LoggerContext context = (LoggerContext) LogManager.getContext(false);
 
         SearchPhaseContext searchPhaseContext1 = new MockSearchPhaseContext(1);
-        ClusterService clusterService1 = new ClusterService(
+        ClusterService clusterService1 = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             null
@@ -99,7 +100,7 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() {
         int numberOfLoggersBefore = context.getLoggers().size();
 
         SearchPhaseContext searchPhaseContext2 = new MockSearchPhaseContext(1);
-        ClusterService clusterService2 = new ClusterService(
+        ClusterService clusterService2 = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             null
@@ -124,7 +125,7 @@ public void testOnRequestEnd() throws InterruptedException {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_THRESHOLD_TRACE_SETTING.getKey(), "0ms");
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService, logger);
         final List<SearchRequestOperationsListener> searchListenersList = new ArrayList<>(List.of(searchRequestSlowLog));
 
@@ -157,7 +158,7 @@ public void testConcurrentOnRequestEnd() throws InterruptedException {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_THRESHOLD_TRACE_SETTING.getKey(), "-1");
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService, logger);
         final List<SearchRequestOperationsListener> searchListenersList = new ArrayList<>(List.of(searchRequestSlowLog));
 
@@ -321,7 +322,7 @@ public void testLevelSettingWarn() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_LEVEL.getKey(), level);
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService);
         assertEquals(level, searchRequestSlowLog.getLevel());
     }
@@ -332,7 +333,7 @@ public void testLevelSettingDebug() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_LEVEL.getKey(), level);
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService);
         assertEquals(level, searchRequestSlowLog.getLevel().toString());
     }
@@ -343,7 +344,7 @@ public void testLevelSettingFail() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_LEVEL.getKey(), level);
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
 
         try {
             new SearchRequestSlowLog(clusterService);
@@ -363,7 +364,7 @@ public void testSetThresholds() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_THRESHOLD_TRACE_SETTING.getKey(), "100ms");
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService);
         assertEquals(TimeValue.timeValueMillis(400).nanos(), searchRequestSlowLog.getWarnThreshold());
         assertEquals(TimeValue.timeValueMillis(300).nanos(), searchRequestSlowLog.getInfoThreshold());
@@ -380,7 +381,7 @@ public void testSetThresholdsUnits() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_THRESHOLD_TRACE_SETTING.getKey(), "100nanos");
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService);
         assertEquals(TimeValue.timeValueSeconds(400).nanos(), searchRequestSlowLog.getWarnThreshold());
         assertEquals(TimeValue.timeValueMillis(300).nanos(), searchRequestSlowLog.getInfoThreshold());
@@ -395,7 +396,7 @@ public void testSetThresholdsDefaults() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_THRESHOLD_DEBUG_SETTING.getKey(), "200ms");
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
         SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService);
         assertEquals(TimeValue.timeValueMillis(400).nanos(), searchRequestSlowLog.getWarnThreshold());
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), searchRequestSlowLog.getInfoThreshold());
@@ -409,7 +410,7 @@ public void testSetThresholdsError() {
         settingsBuilder.put(SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_THRESHOLD_WARN_SETTING.getKey(), "NOT A TIME VALUE");
         Settings settings = settingsBuilder.build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
 
         try {
             new SearchRequestSlowLog(clusterService);
diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
index 557e4dc2ca8c5..ae35d37fe77b2 100644
--- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
@@ -72,6 +72,8 @@
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.gateway.GatewayAllocator;
 import org.opensearch.plugins.ClusterPlugin;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.gateway.TestGatewayAllocator;
 import org.opensearch.test.gateway.TestShardBatchGatewayAllocator;
 
@@ -92,7 +94,7 @@ public class ClusterModuleTests extends ModuleTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadContext = new ThreadContext(Settings.EMPTY);
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             null
@@ -167,7 +169,7 @@ public void testRegisterAllocationDeciderDuplicate() {
                 public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                     return Collections.singletonList(new EnableAllocationDecider(settings, clusterSettings));
                 }
-            }), clusterInfoService, null, threadContext)
+            }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE))
         );
         assertEquals(e.getMessage(), "Cannot specify allocation decider [" + EnableAllocationDecider.class.getName() + "] twice");
     }
@@ -178,7 +180,7 @@ public void testRegisterAllocationDecider() {
             public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                 return Collections.singletonList(new FakeAllocationDecider());
             }
-        }), clusterInfoService, null, threadContext);
+        }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE));
         assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class)));
     }
 
@@ -188,7 +190,7 @@ private ClusterModule newClusterModuleWithShardsAllocator(Settings settings, Str
             public Map<String, Supplier<ShardsAllocator>> getShardsAllocators(Settings settings, ClusterSettings clusterSettings) {
                 return Collections.singletonMap(name, supplier);
             }
-        }), clusterInfoService, null, threadContext);
+        }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE));
     }
 
     public void testRegisterShardsAllocator() {
@@ -209,7 +211,15 @@ public void testUnknownShardsAllocator() {
         Settings settings = Settings.builder().put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), "dne").build();
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> new ClusterModule(settings, clusterService, Collections.emptyList(), clusterInfoService, null, threadContext)
+            () -> new ClusterModule(
+                settings,
+                clusterService,
+                Collections.emptyList(),
+                clusterInfoService,
+                null,
+                threadContext,
+                new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+            )
         );
         assertEquals("Unknown ShardsAllocator [dne]", e.getMessage());
     }
@@ -295,7 +305,8 @@ public void testRejectsReservedExistingShardsAllocatorName() {
             Collections.singletonList(existingShardsAllocatorPlugin(GatewayAllocator.ALLOCATOR_NAME)),
             clusterInfoService,
             null,
-            threadContext
+            threadContext,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         expectThrows(
             IllegalArgumentException.class,
@@ -310,7 +321,8 @@ public void testRejectsDuplicateExistingShardsAllocatorName() {
             Arrays.asList(existingShardsAllocatorPlugin("duplicate"), existingShardsAllocatorPlugin("duplicate")),
             clusterInfoService,
             null,
-            threadContext
+            threadContext,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         expectThrows(
             IllegalArgumentException.class,
diff --git a/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java
index 47dbf85c13b1f..537b2d13ec08a 100644
--- a/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java
+++ b/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java
@@ -54,6 +54,7 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.node.Node;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.client.NoOpClient;
@@ -83,7 +84,13 @@ public void testScheduling() {
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
         final ThreadPool threadPool = deterministicTaskQueue.getThreadPool();
 
-        final ClusterApplierService clusterApplierService = new ClusterApplierService("test", settings, clusterSettings, threadPool) {
+        final ClusterApplierService clusterApplierService = new ClusterApplierService(
+            "test",
+            settings,
+            clusterSettings,
+            threadPool,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        ) {
             @Override
             protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() {
                 return new MockSinglePrioritizingExecutor("mock-executor", deterministicTaskQueue, threadPool);
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java
index d94f3fb304fe2..10d5dceb74f55 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java
@@ -33,6 +33,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.OpenSearchAllocationTestCase;
@@ -61,6 +62,7 @@
 import org.opensearch.monitor.StatusInfo;
 import org.opensearch.node.Node;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
@@ -179,7 +181,8 @@ private void setupRealClusterManagerServiceAndCoordinator(long term, ClusterStat
         ClusterManagerService clusterManagerService = new ClusterManagerService(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test_node").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         AtomicReference<ClusterState> clusterStateRef = new AtomicReference<>(initialState);
         clusterManagerService.setClusterStatePublisher((event, publishListener, ackListener) -> {
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java
index 64d9c243304d8..cce75105dd33f 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterInfo;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.EmptyClusterInfoService;
@@ -56,6 +57,9 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.gateway.GatewayAllocator;
 import org.opensearch.snapshots.EmptySnapshotsInfoService;
+import org.opensearch.telemetry.metrics.Histogram;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.gateway.TestGatewayAllocator;
 
@@ -77,6 +81,12 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
+import static org.mockito.ArgumentMatchers.anyDouble;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class AllocationServiceTests extends OpenSearchTestCase {
 
@@ -137,6 +147,16 @@ public void testAssignsPrimariesInPriorityOrderThenReplicas() {
             .put(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), Integer.MAX_VALUE)
             .build();
         final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
+        final Histogram rerouteHistogram = mock(Histogram.class);
+        final Histogram mockedHistogram = mock(Histogram.class);
+        when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> {
+            String histogramName = (String) invocationOnMock.getArguments()[0];
+            if (histogramName.contains("reroute.latency")) {
+                return rerouteHistogram;
+            }
+            return mockedHistogram;
+        });
         final AllocationService allocationService = new AllocationService(
             new AllocationDeciders(
                 Arrays.asList(
@@ -158,7 +178,8 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
                 }
             },
             new EmptyClusterInfoService(),
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            new ClusterManagerMetrics(metricsRegistry)
         );
 
         final String unrealisticAllocatorName = "unrealistic";
@@ -258,10 +279,18 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
         assertThat(routingTable3.index("mediumPriority").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
         assertTrue(routingTable3.index("lowPriority").allPrimaryShardsActive());
         assertThat(routingTable3.index("invalid").shardsWithState(ShardRoutingState.STARTED), empty());
+
+        verify(rerouteHistogram, times(3)).record(anyDouble());
     }
 
     public void testExplainsNonAllocationOfShardWithUnknownAllocator() {
-        final AllocationService allocationService = new AllocationService(null, null, null, null);
+        final AllocationService allocationService = new AllocationService(
+            null,
+            null,
+            null,
+            null,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        );
         allocationService.setExistingShardsAllocators(
             Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, new TestGatewayAllocator())
         );
diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java
index c5ed505e6bbf2..3cbdfb80067d7 100644
--- a/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java
@@ -35,6 +35,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateObserver;
@@ -51,6 +52,8 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.telemetry.metrics.Histogram;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.test.MockLogAppender;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.junit.annotations.TestLogging;
@@ -74,15 +77,30 @@
 import static org.opensearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyDouble;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
 
 public class ClusterApplierServiceTests extends OpenSearchTestCase {
 
     private static ThreadPool threadPool;
     private TimedClusterApplierService clusterApplierService;
+    private static MetricsRegistry metricsRegistry;
+    private static Histogram applierslatenctHistogram;
+    private static Histogram listenerslatenctHistogram;
 
     @BeforeClass
     public static void createThreadPool() {
         threadPool = new TestThreadPool(ClusterApplierServiceTests.class.getName());
+        metricsRegistry = mock(MetricsRegistry.class);
+        applierslatenctHistogram = mock(Histogram.class);
+        listenerslatenctHistogram = mock(Histogram.class);
     }
 
     @AfterClass
@@ -96,6 +114,13 @@ public static void stopThreadPool() {
     @Before
     public void setUp() throws Exception {
         super.setUp();
+        when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> {
+            String histogramName = (String) invocationOnMock.getArguments()[0];
+            if (histogramName.contains("appliers.latency")) {
+                return applierslatenctHistogram;
+            }
+            return listenerslatenctHistogram;
+        });
         clusterApplierService = createTimedClusterService(true);
     }
 
@@ -110,7 +135,8 @@ private TimedClusterApplierService createTimedClusterService(boolean makeCluster
         TimedClusterApplierService timedClusterApplierService = new TimedClusterApplierService(
             Settings.builder().put("cluster.name", "ClusterApplierServiceTests").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            new ClusterManagerMetrics(metricsRegistry)
         );
         timedClusterApplierService.setNodeConnectionsService(createNoOpNodeConnectionsService());
         timedClusterApplierService.setInitialState(
@@ -194,6 +220,8 @@ public void onFailure(String source, Exception e) {
             });
             assertBusy(mockAppender::assertAllExpectationsMatched);
         }
+        verifyNoInteractions(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     @TestLogging(value = "org.opensearch.cluster.service:WARN", reason = "to ensure that we log cluster state events on WARN level")
@@ -291,6 +319,8 @@ public void onFailure(String source, Exception e) {
             latch.await();
             mockAppender.assertAllExpectationsMatched();
         }
+        verifyNoInteractions(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     public void testLocalNodeClusterManagerListenerCallbacks() {
@@ -329,6 +359,10 @@ public void offClusterManager() {
         setState(timedClusterApplierService, state);
         assertThat(isClusterManager.get(), is(true));
 
+        verify(listenerslatenctHistogram, atLeastOnce()).record(anyDouble(), any());
+        clearInvocations(listenerslatenctHistogram);
+        verifyNoInteractions(applierslatenctHistogram);
+
         timedClusterApplierService.close();
     }
 
@@ -366,6 +400,10 @@ public void offMaster() {
         setState(timedClusterApplierService, state);
         assertThat(isClusterManager.get(), is(false));
 
+        verify(listenerslatenctHistogram, atLeastOnce()).record(anyDouble(), any());
+        clearInvocations(listenerslatenctHistogram);
+        verifyNoInteractions(applierslatenctHistogram);
+
         timedClusterApplierService.close();
     }
 
@@ -405,6 +443,10 @@ public void onFailure(String source, Exception e) {
         latch.await();
         assertNull(error.get());
         assertTrue(applierCalled.get());
+
+        verify(applierslatenctHistogram, atLeastOnce()).record(anyDouble(), any());
+        clearInvocations(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     public void testClusterStateApplierBubblesUpExceptionsInApplier() throws InterruptedException {
@@ -435,6 +477,9 @@ public void onFailure(String source, Exception e) {
         latch.await();
         assertNotNull(error.get());
         assertThat(error.get().getMessage(), containsString("dummy exception"));
+
+        verifyNoInteractions(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     public void testClusterStateApplierBubblesUpExceptionsInSettingsApplier() throws InterruptedException {
@@ -478,6 +523,9 @@ public void onFailure(String source, Exception e) {
         latch.await();
         assertNotNull(error.get());
         assertThat(error.get().getMessage(), containsString("illegal value can't update"));
+
+        verifyNoInteractions(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     public void testClusterStateApplierSwallowsExceptionInListener() throws InterruptedException {
@@ -509,6 +557,9 @@ public void onFailure(String source, Exception e) {
         latch.await();
         assertNull(error.get());
         assertTrue(applierCalled.get());
+
+        verifyNoInteractions(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     public void testClusterStateApplierCanCreateAnObserver() throws InterruptedException {
@@ -565,6 +616,10 @@ public void onFailure(String source, Exception e) {
         latch.await();
         assertNull(error.get());
         assertTrue(applierCalled.get());
+
+        verify(applierslatenctHistogram, atLeastOnce()).record(anyDouble(), any());
+        clearInvocations(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     public void testThreadContext() throws InterruptedException {
@@ -609,6 +664,9 @@ public void onFailure(String source, Exception e) {
         }
 
         latch.await();
+
+        verifyNoInteractions(applierslatenctHistogram);
+        verifyNoInteractions(listenerslatenctHistogram);
     }
 
     static class TimedClusterApplierService extends ClusterApplierService {
@@ -617,8 +675,13 @@ static class TimedClusterApplierService extends ClusterApplierService {
         volatile Long currentTimeOverride = null;
         boolean applicationMayFail;
 
-        TimedClusterApplierService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
-            super("test_node", settings, clusterSettings, threadPool);
+        TimedClusterApplierService(
+            Settings settings,
+            ClusterSettings clusterSettings,
+            ThreadPool threadPool,
+            ClusterManagerMetrics clusterManagerMetrics
+        ) {
+            super("test_node", settings, clusterSettings, threadPool, clusterManagerMetrics);
             this.clusterSettings = clusterSettings;
         }
 
diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterServiceTests.java
index 4d88683826af7..bd12b09d2b983 100644
--- a/server/src/test/java/org/opensearch/cluster/service/ClusterServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/ClusterServiceTests.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.junit.After;
@@ -26,11 +27,11 @@ public void terminateThreadPool() {
 
     public void testDeprecatedGetMasterServiceBWC() {
         try (
-            ClusterService clusterService = new ClusterService(
+            ClusterService clusterService = ClusterServiceUtils.createClusterService(
                 Settings.EMPTY,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 threadPool
-            )
+            );
         ) {
             MasterService masterService = clusterService.getMasterService();
             ClusterManagerService clusterManagerService = clusterService.getClusterManagerService();
diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
index 85f6c129944fa..0ff8d9dc4e7a5 100644
--- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
@@ -40,6 +40,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.AckedClusterStateUpdateTask;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateTaskConfig;
@@ -61,6 +62,9 @@
 import org.opensearch.common.util.concurrent.BaseFuture;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.node.Node;
+import org.opensearch.telemetry.metrics.Histogram;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.test.MockLogAppender;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.junit.annotations.TestLogging;
@@ -95,6 +99,13 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyDouble;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class MasterServiceTests extends OpenSearchTestCase {
 
@@ -125,6 +136,10 @@ public void randomizeCurrentTime() {
     }
 
     private ClusterManagerService createClusterManagerService(boolean makeClusterManager) {
+        return createClusterManagerService(makeClusterManager, NoopMetricsRegistry.INSTANCE);
+    }
+
+    private ClusterManagerService createClusterManagerService(boolean makeClusterManager, MetricsRegistry metricsRegistry) {
         final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
         final ClusterManagerService clusterManagerService = new ClusterManagerService(
             Settings.builder()
@@ -132,7 +147,8 @@ private ClusterManagerService createClusterManagerService(boolean makeClusterMan
                 .put(Node.NODE_NAME_SETTING.getKey(), "test_node")
                 .build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            new ClusterManagerMetrics(metricsRegistry)
         );
         final ClusterState initialClusterState = ClusterState.builder(new ClusterName(MasterServiceTests.class.getSimpleName()))
             .nodes(
@@ -154,7 +170,18 @@ private ClusterManagerService createClusterManagerService(boolean makeClusterMan
     }
 
     public void testClusterManagerAwareExecution() throws Exception {
-        final ClusterManagerService nonClusterManager = createClusterManagerService(false);
+        final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
+        final Histogram clusterStateComputeHistogram = mock(Histogram.class);
+        final Histogram clusterStatePublishHistogram = mock(Histogram.class);
+        when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> {
+            String histogramName = (String) invocationOnMock.getArguments()[0];
+            if (histogramName.contains("cluster.state.new.compute.latency")) {
+                return clusterStateComputeHistogram;
+            }
+            return clusterStatePublishHistogram;
+        });
+
+        final ClusterManagerService nonClusterManager = createClusterManagerService(false, metricsRegistry);
 
         final boolean[] taskFailed = { false };
         final CountDownLatch latch1 = new CountDownLatch(1);
@@ -194,6 +221,8 @@ public void onFailure(String source, Exception e) {
         assertFalse("non-cluster-manager cluster state update task was not executed", taskFailed[0]);
 
         nonClusterManager.close();
+
+        verify(clusterStateComputeHistogram, times(1)).record(anyDouble(), any());
     }
 
     public void testThreadContext() throws InterruptedException {
@@ -1070,7 +1099,8 @@ public void testLongClusterStateUpdateLogging() throws Exception {
                         .put(Node.NODE_NAME_SETTING.getKey(), "test_node")
                         .build(),
                     new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                    threadPool
+                    threadPool,
+                    new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
                 )
             ) {
 
@@ -1246,6 +1276,18 @@ public void testAcking() throws InterruptedException {
         final DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
         final DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
         final DiscoveryNode node3 = new DiscoveryNode("node3", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
+
+        final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
+        final Histogram clusterStateComputeHistogram = mock(Histogram.class);
+        final Histogram clusterStatePublishHistogram = mock(Histogram.class);
+        when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> {
+            String histogramName = (String) invocationOnMock.getArguments()[0];
+            if (histogramName.contains("cluster.state.new.compute.latency")) {
+                return clusterStateComputeHistogram;
+            }
+            return clusterStatePublishHistogram;
+        });
+
         try (
             ClusterManagerService clusterManagerService = new ClusterManagerService(
                 Settings.builder()
@@ -1253,7 +1295,8 @@ public void testAcking() throws InterruptedException {
                     .put(Node.NODE_NAME_SETTING.getKey(), "test_node")
                     .build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadPool
+                threadPool,
+                new ClusterManagerMetrics(metricsRegistry)
             )
         ) {
 
@@ -1372,6 +1415,9 @@ public void onAckTimeout() {
                 latch.await();
             }
         }
+
+        verify(clusterStateComputeHistogram, times(2)).record(anyDouble(), any());
+        verify(clusterStatePublishHistogram, times(1)).record(anyDouble());
     }
 
     public void testDeprecatedMasterServiceUpdateTaskThreadName() {
diff --git a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java
index 1c43bb565ef69..dd2fb51151a5b 100644
--- a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java
+++ b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java
@@ -55,6 +55,7 @@
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.repositories.IndexId;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.Arrays;
@@ -123,7 +124,7 @@ public String getValue(final String value) {
 
             })
         );
-        final ClusterService clusterService = new ClusterService(Settings.EMPTY, clusterSettings, null);
+        final ClusterService clusterService = ClusterServiceUtils.createClusterService(Settings.EMPTY, clusterSettings, null);
         final Metadata.Builder builder = Metadata.builder();
         final Settings settings = Settings.builder().put("foo.old", randomAlphaOfLength(8)).build();
         applySettingsToBuilder.accept(builder, settings);
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayServiceTests.java b/server/src/test/java/org/opensearch/gateway/GatewayServiceTests.java
index c448c4b07e03b..59fb7df5428e2 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayServiceTests.java
@@ -53,6 +53,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.snapshots.EmptySnapshotsInfoService;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.gateway.TestGatewayAllocator;
 import org.hamcrest.Matchers;
@@ -68,7 +69,7 @@
 public class GatewayServiceTests extends OpenSearchTestCase {
 
     private GatewayService createService(final Settings.Builder settings) {
-        final ClusterService clusterService = new ClusterService(
+        final ClusterService clusterService = ClusterServiceUtils.createClusterService(
             Settings.builder().put("cluster.name", "GatewayServiceTests").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             null
diff --git a/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java b/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java
index 0b657c1c9745f..8db7c58bf9503 100644
--- a/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/index/IndexingPressureServiceTests.java
@@ -24,6 +24,7 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.stats.IndexingPressurePerShardStats;
 import org.opensearch.index.stats.IndexingPressureStats;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
@@ -44,7 +45,7 @@ public class IndexingPressureServiceTests extends OpenSearchTestCase {
     @Before
     public void beforeTest() {
         clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        clusterService = new ClusterService(settings, clusterSettings, null);
+        clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
     }
 
     public void testCoordinatingOperationForShardIndexingPressure() {
diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java
index ce719a18898f8..c5ad1370ac75a 100644
--- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java
+++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java
@@ -19,6 +19,7 @@
 import org.opensearch.index.stats.IndexingPressurePerShardStats;
 import org.opensearch.index.stats.IndexingPressureStats;
 import org.opensearch.index.stats.ShardIndexingPressureStats;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.hamcrest.Matcher;
 import org.hamcrest.MatcherAssert;
@@ -42,7 +43,7 @@ public class ShardIndexingPressureConcurrentExecutionTests extends OpenSearchTes
         .build();
 
     private final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-    private final ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+    private final ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
 
     public enum OperationType {
         COORDINATING,
diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java
index 023063c7d6e03..31ecad7c8d701 100644
--- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureMemoryManagerTests.java
@@ -8,11 +8,11 @@
 
 package org.opensearch.index;
 
-import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.concurrent.TimeUnit;
@@ -27,7 +27,7 @@ public class ShardIndexingPressureMemoryManagerTests extends OpenSearchTestCase
         .build();
     private final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
     private final ShardIndexingPressureSettings shardIndexingPressureSettings = new ShardIndexingPressureSettings(
-        new ClusterService(settings, clusterSettings, null),
+        ClusterServiceUtils.createClusterService(settings, clusterSettings, null),
         settings,
         IndexingPressure.MAX_INDEXING_BYTES.get(settings).getBytes()
     );
diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureSettingsTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureSettingsTests.java
index c555d8f9c489d..5e84a76b2250a 100644
--- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureSettingsTests.java
+++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureSettingsTests.java
@@ -11,6 +11,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 
 public class ShardIndexingPressureSettingsTests extends OpenSearchTestCase {
@@ -24,7 +25,7 @@ public class ShardIndexingPressureSettingsTests extends OpenSearchTestCase {
         .build();
 
     final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-    final ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+    final ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
 
     public void testFromSettings() {
         ShardIndexingPressureSettings shardIndexingPressureSettings = new ShardIndexingPressureSettings(
diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java
index 46f9801035ac3..d97eec4cc001d 100644
--- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java
+++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureStoreTests.java
@@ -8,10 +8,10 @@
 
 package org.opensearch.index;
 
-import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
@@ -22,7 +22,7 @@ public class ShardIndexingPressureStoreTests extends OpenSearchTestCase {
     private final Settings settings = Settings.builder().put(ShardIndexingPressureStore.MAX_COLD_STORE_SIZE.getKey(), 200).build();
     private final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
     private final ShardIndexingPressureSettings shardIndexingPressureSettings = new ShardIndexingPressureSettings(
-        new ClusterService(settings, clusterSettings, null),
+        ClusterServiceUtils.createClusterService(settings, clusterSettings, null),
         settings,
         IndexingPressure.MAX_INDEXING_BYTES.get(settings).getBytes()
     );
diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java
index e7600b1d4c41a..ddc3592511de4 100644
--- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java
+++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureTests.java
@@ -17,6 +17,7 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.stats.IndexingPressurePerShardStats;
 import org.opensearch.index.stats.IndexingPressureStats;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 
 public class ShardIndexingPressureTests extends OpenSearchTestCase {
@@ -30,7 +31,7 @@ public class ShardIndexingPressureTests extends OpenSearchTestCase {
         .build();
 
     final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-    final ClusterService clusterService = new ClusterService(settings, clusterSettings, null);
+    final ClusterService clusterService = ClusterServiceUtils.createClusterService(settings, clusterSettings, null);
 
     public void testMemoryBytesMarkedAndReleased() {
         ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
index ccdd1fe4ab609..280598c516c3c 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
@@ -15,6 +15,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.store.DirectoryFileTransferTracker;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -42,7 +43,7 @@ public class RemoteSegmentTransferTrackerTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("remote_refresh_segment_pressure_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java
index 9d00cf9f2be46..18d18f2dc30b1 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java
@@ -14,6 +14,7 @@
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.shard.IndexShard;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -45,7 +46,7 @@ public class RemoteStorePressureServiceTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("remote_refresh_segment_pressure_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java
index 064c6c10eba02..7c1ef0de91887 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java
@@ -11,6 +11,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -27,7 +28,7 @@ public class RemoteStorePressureSettingsTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("remote_refresh_segment_pressure_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactoryTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactoryTests.java
index c300f316ac633..2bc4792a9c31c 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactoryTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactoryTests.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.shard.IndexShard;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -39,7 +40,11 @@ public void setUp() throws Exception {
                 RemoteStoreStatsTrackerFactory.Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE
             )
             .build();
-        clusterService = new ClusterService(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool);
+        clusterService = ClusterServiceUtils.createClusterService(
+            settings,
+            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool
+        );
         remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings);
     }
 
@@ -85,7 +90,11 @@ public void testInvalidMovingAverageWindowSize() {
             "Failed to parse value",
             IllegalArgumentException.class,
             () -> new RemoteStoreStatsTrackerFactory(
-                new ClusterService(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool),
+                ClusterServiceUtils.createClusterService(
+                    settings,
+                    new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                    threadPool
+                ),
                 settings
             )
         );
@@ -107,7 +116,11 @@ public void testUpdateAfterGetConfiguredSettings() {
 
     public void testGetDefaultSettings() {
         remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(
-            new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool),
+            ClusterServiceUtils.createClusterService(
+                Settings.EMPTY,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                threadPool
+            ),
             Settings.EMPTY
         );
         // Check moving average window size updated
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
index 33f6c67b94b3d..8fbff4527ec7b 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
@@ -37,6 +37,7 @@
 import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
 import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.threadpool.ThreadPool;
 import org.junit.After;
 
@@ -82,7 +83,7 @@ public void setup(boolean primary, int numberOfDocs) throws IOException {
             indexShard.refresh("test");
         }
 
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
@@ -627,7 +628,7 @@ private Tuple<RemoteStoreRefreshListener, RemoteStoreStatsTrackerFactory> mockIn
             return null;
         }).when(emptyCheckpointPublisher).publish(any(), any());
 
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/node/ResponseCollectorServiceTests.java b/server/src/test/java/org/opensearch/node/ResponseCollectorServiceTests.java
index 7ca1f1e864b99..7567224f16ac3 100644
--- a/server/src/test/java/org/opensearch/node/ResponseCollectorServiceTests.java
+++ b/server/src/test/java/org/opensearch/node/ResponseCollectorServiceTests.java
@@ -41,6 +41,7 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -63,7 +64,7 @@ public class ResponseCollectorServiceTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadpool = new TestThreadPool("response_collector_tests");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadpool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java
index 4f615290f1805..fbb083a3ae419 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java
@@ -17,6 +17,7 @@
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
 import org.opensearch.ratelimitting.admissioncontrol.settings.CpuBasedAdmissionControllerSettings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -33,7 +34,7 @@ public class AdmissionControlServiceTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java
index c11ee1cc608f6..fbadcad804b31 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -28,7 +29,7 @@ public class AdmissionControlSettingsTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CpuBasedAdmissionControllerTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CpuBasedAdmissionControllerTests.java
index e72c0cd58ed64..f2cb45a033460 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CpuBasedAdmissionControllerTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CpuBasedAdmissionControllerTests.java
@@ -15,6 +15,7 @@
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
 import org.opensearch.ratelimitting.admissioncontrol.settings.CpuBasedAdmissionControllerSettings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -31,7 +32,7 @@ public class CpuBasedAdmissionControllerTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/IoBasedAdmissionControllerTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/IoBasedAdmissionControllerTests.java
index c5a2208f49ce6..54cb438e14ce6 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/IoBasedAdmissionControllerTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/IoBasedAdmissionControllerTests.java
@@ -15,6 +15,7 @@
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
 import org.opensearch.ratelimitting.admissioncontrol.settings.IoBasedAdmissionControllerSettings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -31,7 +32,7 @@ public class IoBasedAdmissionControllerTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettingsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettingsTests.java
index 6836ecb3d615f..f5686f33e7f50 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettingsTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettingsTests.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -28,7 +29,7 @@ public class CPUBasedAdmissionControllerSettingsTests extends OpenSearchTestCase
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/IoBasedAdmissionControllerSettingsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/IoBasedAdmissionControllerSettingsTests.java
index c462f9700264d..3f157531f6c9a 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/IoBasedAdmissionControllerSettingsTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/IoBasedAdmissionControllerSettingsTests.java
@@ -21,6 +21,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -36,7 +37,7 @@ public class IoBasedAdmissionControllerSettingsTests extends OpenSearchTestCase
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool("io_based_admission_controller_settings_test");
-        clusterService = new ClusterService(
+        clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControlStatsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControlStatsTests.java
index 7b4db5f787d6e..da57ef9f06a1a 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControlStatsTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControlStatsTests.java
@@ -20,6 +20,7 @@
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
 import org.opensearch.ratelimitting.admissioncontrol.settings.CpuBasedAdmissionControllerSettings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -46,7 +47,7 @@ public void setUp() throws Exception {
             )
             .build();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        ClusterService clusterService = new ClusterService(
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControllerStatsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControllerStatsTests.java
index fe0399e79a5f4..0ef9aa61bb827 100644
--- a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControllerStatsTests.java
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/stats/AdmissionControllerStatsTests.java
@@ -20,6 +20,7 @@
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
 import org.opensearch.ratelimitting.admissioncontrol.settings.CpuBasedAdmissionControllerSettings;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -43,7 +44,7 @@ public void setUp() throws Exception {
             )
             .build();
         threadPool = new TestThreadPool("admission_controller_settings_test");
-        ClusterService clusterService = new ClusterService(
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index 95a343f3b4025..3a07aa413e2a2 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -112,6 +112,7 @@
 import org.opensearch.cluster.ClusterChangedEvent;
 import org.opensearch.cluster.ClusterInfo;
 import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
@@ -1922,7 +1923,13 @@ private final class TestClusterNode {
                     settings,
                     clusterSettings,
                     clusterManagerService,
-                    new ClusterApplierService(node.getName(), settings, clusterSettings, threadPool) {
+                    new ClusterApplierService(
+                        node.getName(),
+                        settings,
+                        clusterSettings,
+                        threadPool,
+                        new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+                    ) {
                         @Override
                         protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() {
                             return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool);
diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
index 0754cc1793dc8..3ae737bf63923 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -39,6 +39,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateTaskListener;
@@ -89,6 +90,7 @@
 import org.opensearch.monitor.StatusInfo;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.disruption.DisruptableMockTransport;
@@ -1144,7 +1146,8 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     settings,
                     clusterSettings,
                     deterministicTaskQueue,
-                    threadPool
+                    threadPool,
+                    new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
                 );
                 clusterService = new ClusterService(settings, clusterSettings, clusterManagerService, clusterApplierService);
                 clusterService.setNodeConnectionsService(
@@ -1594,9 +1597,10 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
             Settings settings,
             ClusterSettings clusterSettings,
             DeterministicTaskQueue deterministicTaskQueue,
-            ThreadPool threadPool
+            ThreadPool threadPool,
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
-            super(nodeName, settings, clusterSettings, threadPool);
+            super(nodeName, settings, clusterSettings, threadPool, clusterManagerMetrics);
             this.nodeName = nodeName;
             this.deterministicTaskQueue = deterministicTaskQueue;
             addStateApplier(event -> {
diff --git a/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java b/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java
index 3ca938c99b5fd..53ef595c7931e 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java
@@ -34,6 +34,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.ClusterStatePublisher.AckListener;
 import org.opensearch.common.UUIDs;
@@ -45,6 +46,7 @@
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.node.Node;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
@@ -74,7 +76,8 @@ public FakeThreadPoolClusterManagerService(
         super(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         this.name = serviceName;
         this.onTaskAvailableToRun = onTaskAvailableToRun;
diff --git a/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java
index 8f4f510da5ec3..f0c0e9bc2d589 100644
--- a/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java
@@ -34,6 +34,7 @@
 import org.apache.logging.log4j.core.util.Throwables;
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateUpdateTask;
@@ -52,6 +53,8 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.node.Node;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.util.Collections;
@@ -66,7 +69,8 @@ public static ClusterManagerService createClusterManagerService(ThreadPool threa
         ClusterManagerService clusterManagerService = new ClusterManagerService(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test_cluster_manager_node").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool
+            threadPool,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         AtomicReference<ClusterState> clusterStateRef = new AtomicReference<>(initialClusterState);
         clusterManagerService.setClusterStatePublisher((event, publishListener, ackListener) -> {
@@ -169,8 +173,22 @@ public static ClusterService createClusterService(ThreadPool threadPool, Discove
     }
 
     public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode, ClusterSettings clusterSettings) {
+        return createClusterService(threadPool, localNode, clusterSettings, NoopMetricsRegistry.INSTANCE);
+    }
+
+    public static ClusterService createClusterService(
+        ThreadPool threadPool,
+        DiscoveryNode localNode,
+        ClusterSettings clusterSettings,
+        MetricsRegistry metricsRegistry
+    ) {
         Settings settings = Settings.builder().put("node.name", "test").put("cluster.name", "ClusterServiceTests").build();
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, threadPool);
+        ClusterService clusterService = new ClusterService(
+            settings,
+            clusterSettings,
+            threadPool,
+            new ClusterManagerMetrics(metricsRegistry)
+        );
         clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService());
         ClusterState initialClusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName()))
             .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).clusterManagerNodeId(localNode.getId()))
@@ -184,6 +202,10 @@ public static ClusterService createClusterService(ThreadPool threadPool, Discove
         return clusterService;
     }
 
+    public static ClusterService createClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+        return new ClusterService(settings, clusterSettings, threadPool, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE));
+    }
+
     public static NodeConnectionsService createNoOpNodeConnectionsService() {
         return new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override

From 538cd9f22cee3a7046a70f99e802ff347ca60afa Mon Sep 17 00:00:00 2001
From: Kiran Prakash <awskiran@amazon.com>
Date: Mon, 20 May 2024 12:52:03 -0700
Subject: [PATCH 45/52] Update IndicesRequestCacheIT.java (#13678)

Signed-off-by: Kiran Prakash <awskiran@amazon.com>
---
 .../indices/IndicesRequestCacheIT.java        | 27 -------------------
 1 file changed, 27 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index ae2295cb874f5..766ca2c1189e5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -1077,24 +1077,6 @@ public void testDynamicStalenessThresholdUpdate() throws Exception {
 
     // staleness threshold dynamic updates should throw exceptions on invalid input
     public void testInvalidStalenessThresholdUpdateThrowsException() throws Exception {
-        int cacheCleanIntervalInMillis = 1;
-        String node = internalCluster().startNode(
-            Settings.builder()
-                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
-                .put(
-                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
-                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
-                )
-        );
-        Client client = client(node);
-        String index1 = "index1";
-        setupIndex(client, index1);
-
-        // create first cache entry in index1
-        createCacheEntry(client, index1, "hello");
-        assertCacheState(client, index1, 0, 1);
-        assertTrue(getRequestCacheStats(client, index1).getMemorySizeInBytes() > 0);
-
         // Update indices.requests.cache.cleanup.staleness_threshold to "10%" with illegal argument
         assertThrows("Ratio should be in [0-1.0]", IllegalArgumentException.class, () -> {
             ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
@@ -1103,15 +1085,6 @@ public void testInvalidStalenessThresholdUpdateThrowsException() throws Exceptio
             );
             client().admin().cluster().updateSettings(updateSettingsRequest).actionGet();
         });
-
-        // everything else should continue to work fine later on.
-        // force refresh so that it creates 1 stale key
-        flushAndRefresh(index1);
-        // sleep until cache cleaner would have cleaned up the stale key from index 2
-        assertBusy(() -> {
-            // cache cleaner should NOT have cleaned from index 1
-            assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes());
-        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
     }
 
     // closing the Index after caching will clean up from Indices Request Cache

From 3fe66747b1523c74e06bc3465ef2ab682048f76b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 16:22:10 -0700
Subject: [PATCH 46/52] Bump com.google.code.gson:gson from 2.10.1 to 2.11.0 in
 /plugins/repository-hdfs (#13752)

* Bump com.google.code.gson:gson in /plugins/repository-hdfs

Bumps [com.google.code.gson:gson](https://github.com/google/gson) from 2.10.1 to 2.11.0.
- [Release notes](https://github.com/google/gson/releases)
- [Changelog](https://github.com/google/gson/blob/main/CHANGELOG.md)
- [Commits](https://github.com/google/gson/compare/gson-parent-2.10.1...gson-parent-2.11.0)

---
updated-dependencies:
- dependency-name: com.google.code.gson:gson
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* Updating SHAs

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                                          | 1 +
 plugins/repository-hdfs/build.gradle                  | 2 +-
 plugins/repository-hdfs/licenses/gson-2.10.1.jar.sha1 | 1 -
 plugins/repository-hdfs/licenses/gson-2.11.0.jar.sha1 | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/repository-hdfs/licenses/gson-2.10.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/gson-2.11.0.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5060409cec3db..2c905def74219 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
 - Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
+- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#13752](https://github.com/opensearch-project/OpenSearch/pull/13752))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index e019a878dfcf0..d97ff72cf2ebb 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -67,7 +67,7 @@ dependencies {
   api 'org.apache.htrace:htrace-core4:4.2.0-incubating'
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   api 'org.apache.avro:avro:1.11.3'
-  api 'com.google.code.gson:gson:2.10.1'
+  api 'com.google.code.gson:gson:2.11.0'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api 'commons-cli:commons-cli:1.7.0'
diff --git a/plugins/repository-hdfs/licenses/gson-2.10.1.jar.sha1 b/plugins/repository-hdfs/licenses/gson-2.10.1.jar.sha1
deleted file mode 100644
index 9810309d1013a..0000000000000
--- a/plugins/repository-hdfs/licenses/gson-2.10.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b3add478d4382b78ea20b1671390a858002feb6c
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/gson-2.11.0.jar.sha1 b/plugins/repository-hdfs/licenses/gson-2.11.0.jar.sha1
new file mode 100644
index 0000000000000..0414a49526895
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/gson-2.11.0.jar.sha1
@@ -0,0 +1 @@
+527175ca6d81050b53bdd4c457a6d6e017626b0e
\ No newline at end of file

From 04e28fae9506679c82c442d0c53e358f85bc22b7 Mon Sep 17 00:00:00 2001
From: Rishab Nahata <rnnahata@amazon.com>
Date: Tue, 21 May 2024 15:00:32 +0530
Subject: [PATCH 47/52] Skip unnecessary string format in
 ThrottlingAllocationDecider when not in debug mode (#13750)

* Skip unnecessary String format in throttling allocation decider when not in debug mode

Signed-off-by: Rishab Nahata <rnnahata@amazon.com>
---
 .../decider/ThrottlingAllocationDecider.java  | 95 ++++++++++---------
 1 file changed, 49 insertions(+), 46 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index 26a04de31ce39..61e7aaed5ecff 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -44,7 +44,6 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 
-import java.util.Locale;
 import java.util.function.BiFunction;
 
 import static org.opensearch.cluster.routing.allocation.decider.Decision.THROTTLE;
@@ -211,20 +210,9 @@ private Decision allocateInitialShardCopies(ShardRouting shardRouting, RoutingNo
             allocation,
             currentInRecoveries,
             replicasInitialRecoveries,
-            (x, y) -> getInitialPrimaryNodeOutgoingRecoveries(x, y),
+            this::getInitialPrimaryNodeOutgoingRecoveries,
             replicasInitialRecoveries,
-            String.format(
-                Locale.ROOT,
-                "[%s=%d]",
-                CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(),
-                replicasInitialRecoveries
-            ),
-            String.format(
-                Locale.ROOT,
-                "[%s=%d]",
-                CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(),
-                replicasInitialRecoveries
-            )
+            true
         );
     }
 
@@ -238,22 +226,9 @@ private Decision allocateNonInitialShardCopies(ShardRouting shardRouting, Routin
             allocation,
             currentInRecoveries,
             concurrentIncomingRecoveries,
-            (x, y) -> getPrimaryNodeOutgoingRecoveries(x, y),
+            this::getPrimaryNodeOutgoingRecoveries,
             concurrentOutgoingRecoveries,
-            String.format(
-                Locale.ROOT,
-                "[%s=%d] (can also be set via [%s])",
-                CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(),
-                concurrentIncomingRecoveries,
-                CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey()
-            ),
-            String.format(
-                Locale.ROOT,
-                "[%s=%d] (can also be set via [%s])",
-                CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(),
-                concurrentOutgoingRecoveries,
-                CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey()
-            )
+            false
         );
     }
 
@@ -274,18 +249,30 @@ private Decision allocateShardCopies(
         int inRecoveriesLimit,
         BiFunction<ShardRouting, RoutingAllocation, Integer> primaryNodeOutRecoveriesFunc,
         int outRecoveriesLimit,
-        String incomingRecoveriesSettingMsg,
-        String outGoingRecoveriesSettingMsg
+        boolean isInitialShardCopies
     ) {
         // Allocating a shard to this node will increase the incoming recoveries
         if (currentInRecoveries >= inRecoveriesLimit) {
-            return allocation.decision(
-                THROTTLE,
-                NAME,
-                "reached the limit of incoming shard recoveries [%d], cluster setting %s",
-                currentInRecoveries,
-                incomingRecoveriesSettingMsg
-            );
+            if (isInitialShardCopies) {
+                return allocation.decision(
+                    THROTTLE,
+                    NAME,
+                    "reached the limit of incoming shard recoveries [%d], cluster setting [%s=%d]",
+                    currentInRecoveries,
+                    CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(),
+                    inRecoveriesLimit
+                );
+            } else {
+                return allocation.decision(
+                    THROTTLE,
+                    NAME,
+                    "reached the limit of incoming shard recoveries [%d], cluster setting [%s=%d] (can also be set via [%s])",
+                    currentInRecoveries,
+                    CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(),
+                    inRecoveriesLimit,
+                    CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey()
+                );
+            }
         } else {
             // search for corresponding recovery source (= primary shard) and check number of outgoing recoveries on that node
             ShardRouting primaryShard = allocation.routingNodes().activePrimary(shardRouting.shardId());
@@ -294,14 +281,30 @@ private Decision allocateShardCopies(
             }
             int primaryNodeOutRecoveries = primaryNodeOutRecoveriesFunc.apply(shardRouting, allocation);
             if (primaryNodeOutRecoveries >= outRecoveriesLimit) {
-                return allocation.decision(
-                    THROTTLE,
-                    NAME,
-                    "reached the limit of outgoing shard recoveries [%d] on the node [%s] which holds the primary, " + "cluster setting %s",
-                    primaryNodeOutRecoveries,
-                    primaryShard.currentNodeId(),
-                    outGoingRecoveriesSettingMsg
-                );
+                if (isInitialShardCopies) {
+                    return allocation.decision(
+                        THROTTLE,
+                        NAME,
+                        "reached the limit of outgoing shard recoveries [%d] on the node [%s] which holds the primary, "
+                            + "cluster setting [%s=%d]",
+                        primaryNodeOutRecoveries,
+                        primaryShard.currentNodeId(),
+                        CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(),
+                        inRecoveriesLimit
+                    );
+                } else {
+                    return allocation.decision(
+                        THROTTLE,
+                        NAME,
+                        "reached the limit of outgoing shard recoveries [%d] on the node [%s] which holds the primary, "
+                            + "cluster setting [%s=%d] (can also be set via [%s])",
+                        primaryNodeOutRecoveries,
+                        primaryShard.currentNodeId(),
+                        CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(),
+                        outRecoveriesLimit,
+                        CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey()
+                    );
+                }
             } else {
                 return allocation.decision(
                     YES,

From 4fbcedd77fe45802234963893039d02b802461cb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 21 May 2024 10:19:10 -0400
Subject: [PATCH 48/52] Bump ch.qos.logback:logback-core from 1.5.3 to 1.5.6 in
 /test/fixtures/hdfs-fixture (#13756)

* Bump ch.qos.logback:logback-core in /test/fixtures/hdfs-fixture

Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.5.3 to 1.5.6.
- [Commits](https://github.com/qos-ch/logback/compare/v_1.5.3...v_1.5.6)

---
updated-dependencies:
- dependency-name: ch.qos.logback:logback-core
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md                            | 1 +
 test/fixtures/hdfs-fixture/build.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2c905def74219..b270949c2c416 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
 - Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
 - Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#13752](https://github.com/opensearch-project/OpenSearch/pull/13752))
+- Bump `ch.qos.logback:logback-core` from 1.5.3 to 1.5.6 ([#13756](https://github.com/opensearch-project/OpenSearch/pull/13756))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 33635f2053a21..d8d5d95651b5e 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -71,7 +71,7 @@ dependencies {
   api 'org.apache.zookeeper:zookeeper:3.9.2'
   api "org.apache.commons:commons-text:1.12.0"
   api "commons-net:commons-net:3.10.0"
-  api "ch.qos.logback:logback-core:1.5.3"
+  api "ch.qos.logback:logback-core:1.5.6"
   api "ch.qos.logback:logback-classic:1.2.13"
   api 'org.apache.kerby:kerb-admin:2.0.3'
   runtimeOnly "com.google.guava:guava:${versions.guava}"

From 46666c676893134884a147936ff35801f6f49f74 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 21 May 2024 13:56:00 -0400
Subject: [PATCH 49/52] Bump com.gradle.enterprise from 3.17.3 to 3.17.4
 (#13753)

* Bump com.gradle.enterprise from 3.17.3 to 3.17.4

Bumps com.gradle.enterprise from 3.17.3 to 3.17.4.

---
updated-dependencies:
- dependency-name: com.gradle.enterprise
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update changelog

Signed-off-by: dependabot[bot] <support@github.com>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <dependabot[bot]@users.noreply.github.com>
---
 CHANGELOG.md    | 2 +-
 settings.gradle | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b270949c2c416..423d23599af3b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
 - Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
-- Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
+- Bump `com.gradle.enterprise` from 3.17.2 to 3.17.4 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641), [#13753](https://github.com/opensearch-project/OpenSearch/pull/13753))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
 - Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
 - Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#13752](https://github.com/opensearch-project/OpenSearch/pull/13752))
diff --git a/settings.gradle b/settings.gradle
index 065a8dc3d0a8a..f58a8b35b10eb 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -10,7 +10,7 @@
  */
 
 plugins {
-  id "com.gradle.enterprise" version "3.17.3"
+  id "com.gradle.enterprise" version "3.17.4"
 }
 
 ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE')

From 199a03e62558e005d91f4aeb5875573fc564987f Mon Sep 17 00:00:00 2001
From: Sorabh <sohami.apache@gmail.com>
Date: Tue, 21 May 2024 19:15:03 -0700
Subject: [PATCH 50/52] Add Jay Deng as a maintainer (#13773)

Signed-off-by: Sorabh Hamirwasia <sohami.apache@gmail.com>
---
 .github/CODEOWNERS | 4 ++--
 MAINTAINERS.md     | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 40c8fb2e01dd7..b5b0a815b02b2 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,5 +1,5 @@
 # CODEOWNERS manages notifications, not PR approvals
-# For PR approvals see /.github/workflows/maintainer-approval.yml 
+# For PR approvals see /.github/workflows/maintainer-approval.yml
 
 # Files have a single rule applied, the last match decides the owner
 # If you would like to more specifically apply ownership, include existing owner in new sub fields
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index cce92167473b6..6855281a488ca 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -14,6 +14,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)                     | Amazon      |
 | Gaurav Bafna             | [gbbafna](https://github.com/gbbafna)                   | Amazon      |
+| Jay Deng                 | [jed326](https://github.com/jed326)                     | Amazon      |
 | Kunal Kotwani            | [kotwanikunal](https://github.com/kotwanikunal)         | Amazon      |
 | Marc Handalian           | [mch2](https://github.com/mch2)                         | Amazon      |
 | Michael Froh             | [msfroh](https://github.com/msfroh)                     | Amazon      |

From 66df9303f7979cdb0ee6d4e3073c41883f22b5cb Mon Sep 17 00:00:00 2001
From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com>
Date: Wed, 22 May 2024 14:09:53 +0530
Subject: [PATCH 51/52] de-duping shards in ShardsBatchGatewayAllocator based
 on ShardId instead of ShardRouting#equals (#13710)

Signed-off-by: Rajiv Kumar Vaidyanathan <rajivkv@amazon.com>
---
 CHANGELOG.md                                  |  1 +
 .../gateway/ShardsBatchGatewayAllocator.java  | 13 ++--
 .../gateway/GatewayAllocatorTests.java        | 75 +++++++++++++++++++
 .../TestShardBatchGatewayAllocator.java       |  8 ++
 4 files changed, 92 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 423d23599af3b..a4e920f3a202d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 - Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
 - Allow clearing `remote_store.compatibility_mode` setting ([#13646](https://github.com/opensearch-project/OpenSearch/pull/13646))
+- Fix ReplicaShardBatchAllocator to batch shards without duplicates ([#13710](https://github.com/opensearch-project/OpenSearch/pull/13710))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
index d2de78ffac965..3c0797cd450d2 100644
--- a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
@@ -117,14 +117,17 @@ public void cleanCaches() {
 
     // for tests
     protected ShardsBatchGatewayAllocator() {
+        this(DEFAULT_SHARD_BATCH_SIZE);
+    }
+
+    protected ShardsBatchGatewayAllocator(long batchSize) {
         this.rerouteService = null;
         this.batchStartedAction = null;
         this.primaryShardBatchAllocator = null;
         this.batchStoreAction = null;
         this.replicaShardBatchAllocator = null;
-        this.maxBatchSize = DEFAULT_SHARD_BATCH_SIZE;
+        this.maxBatchSize = batchSize;
     }
-
     // for tests
 
     @Override
@@ -228,13 +231,13 @@ protected Set<String> createAndUpdateBatches(RoutingAllocation allocation, boole
             batchEntry.getValue().getBatchedShards().forEach(shardId -> currentBatchedShards.put(shardId, batchEntry.getKey()));
         }
 
-        Set<ShardRouting> newShardsToBatch = Sets.newHashSet();
+        Map<ShardId, ShardRouting> newShardsToBatch = new HashMap<>();
         Set<ShardId> batchedShardsToAssign = Sets.newHashSet();
         // add all unassigned shards to the batch if they are not already in a batch
         unassigned.forEach(shardRouting -> {
             if ((currentBatchedShards.containsKey(shardRouting.shardId()) == false) && (shardRouting.primary() == primary)) {
                 assert shardRouting.unassigned();
-                newShardsToBatch.add(shardRouting);
+                newShardsToBatch.put(shardRouting.shardId(), shardRouting);
             }
             // if shard is already batched update to latest shardRouting information in the batches
             // Replica shard assignment can be cancelled if we get a better match. These ShardRouting objects also
@@ -262,7 +265,7 @@ else if (shardRouting.primary() == primary) {
 
         refreshShardBatches(currentBatches, batchedShardsToAssign, primary);
 
-        Iterator<ShardRouting> iterator = newShardsToBatch.iterator();
+        Iterator<ShardRouting> iterator = newShardsToBatch.values().iterator();
         assert maxBatchSize > 0 : "Shards batch size must be greater than 0";
 
         long batchSize = maxBatchSize;
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java
index bb59a5792ec8c..aa31c710c1fbd 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java
@@ -18,14 +18,21 @@
 import org.opensearch.cluster.OpenSearchAllocationTestCase;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingNodes;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.TestShardRouting;
+import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.snapshots.SnapshotShardSizeInfo;
 import org.opensearch.test.gateway.TestShardBatchGatewayAllocator;
@@ -222,6 +229,21 @@ public void testSafelyRemoveShardFromBothBatch() {
         assertEquals(0, testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().size());
     }
 
+    public void testDeDuplicationOfReplicaShardsAcrossBatch() {
+        final ShardId shardId = new ShardId("test", "_na_", 0);
+        final DiscoveryNode node = newNode("node1");
+        // number of replicas is greater than batch size - to ensure shardRouting gets de-duped across batch
+        createRoutingWithDifferentUnAssignedInfo(shardId, node, 50);
+        testShardsBatchGatewayAllocator = new TestShardBatchGatewayAllocator(10);
+
+        // only replica shard should be in the batch
+        Set<String> replicaBatches = testShardsBatchGatewayAllocator.createAndUpdateBatches(testAllocation, false);
+        assertEquals(1, replicaBatches.size());
+        ShardsBatchGatewayAllocator.ShardsBatch shardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch()
+            .get(replicaBatches.iterator().next());
+        assertEquals(1, shardsBatch.getBatchedShards().size());
+    }
+
     public void testGetBatchIdExisting() {
         createIndexAndUpdateClusterState(2, 1020, 1);
         // get all shardsRoutings for test index
@@ -345,6 +367,59 @@ private void createIndexAndUpdateClusterState(int count, int numberOfShards, int
         );
     }
 
+    private void createRoutingWithDifferentUnAssignedInfo(ShardId primaryShardId, DiscoveryNode node, int numberOfReplicas) {
+
+        ShardRouting primaryShard = TestShardRouting.newShardRouting(primaryShardId, node.getId(), true, ShardRoutingState.STARTED);
+        Metadata metadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder(primaryShardId.getIndexName())
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(1)
+                    .numberOfReplicas(numberOfReplicas)
+                    .putInSyncAllocationIds(0, Sets.newHashSet(primaryShard.allocationId().getId()))
+            )
+            .build();
+
+        IndexRoutingTable.Builder isd = IndexRoutingTable.builder(primaryShardId.getIndex())
+            .addIndexShard(new IndexShardRoutingTable.Builder(primaryShardId).addShard(primaryShard).build());
+
+        for (int i = 0; i < numberOfReplicas; i++) {
+            isd.addShard(
+                ShardRouting.newUnassigned(
+                    primaryShardId,
+                    false,
+                    RecoverySource.PeerRecoverySource.INSTANCE,
+                    new UnassignedInfo(
+                        UnassignedInfo.Reason.REPLICA_ADDED,
+                        "message for replica-copy " + i,
+                        null,
+                        0,
+                        System.nanoTime(),
+                        System.currentTimeMillis(),
+                        false,
+                        UnassignedInfo.AllocationStatus.NO_ATTEMPT,
+                        Collections.emptySet()
+                    )
+                )
+            );
+        }
+
+        RoutingTable routingTable = RoutingTable.builder().add(isd).build();
+        clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .routingTable(routingTable)
+            .build();
+        testAllocation = new RoutingAllocation(
+            new AllocationDeciders(Collections.emptyList()),
+            new RoutingNodes(clusterState, false),
+            clusterState,
+            ClusterInfo.EMPTY,
+            SnapshotShardSizeInfo.EMPTY,
+            System.nanoTime()
+        );
+
+    }
+
     // call this after index creation and update cluster state
     private Tuple<Set<String>, Set<String>> createBatchesAndAssert(int expectedBatchSize) {
         Set<String> primaryBatches = testShardsBatchGatewayAllocator.createAndUpdateBatches(testAllocation, true);
diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java
index 53a4e90adb976..fbb39c284f0ff 100644
--- a/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java
+++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java
@@ -31,6 +31,14 @@
 
 public class TestShardBatchGatewayAllocator extends ShardsBatchGatewayAllocator {
 
+    public TestShardBatchGatewayAllocator() {
+
+    }
+
+    public TestShardBatchGatewayAllocator(long maxBatchSize) {
+        super(maxBatchSize);
+    }
+
     Map<String /* node id */, Map<ShardId, ShardRouting>> knownAllocations = new HashMap<>();
     DiscoveryNodes currentNodes = DiscoveryNodes.EMPTY_NODES;
     Map<String, ReplicationCheckpoint> shardIdNodeToReplicationCheckPointMap = new HashMap<>();

From 5441d55ee191ca789ba86b7e477a64eb73398c80 Mon Sep 17 00:00:00 2001
From: Prudhvi Godithi <pgodithi@amazon.com>
Date: Wed, 22 May 2024 14:29:43 -0700
Subject: [PATCH 52/52] Gradle Check Optimization (#13786)

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>
---
 .github/workflows/gradle-check.yml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
index 6d8fe02c7c9c3..f610ff5c31049 100644
--- a/.github/workflows/gradle-check.yml
+++ b/.github/workflows/gradle-check.yml
@@ -39,6 +39,22 @@ jobs:
           echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_owner=$(jq --raw-output .pull_request.user.login $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_or_commit_description=$(jq --ascii-output .pull_request.body $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "post_merge_action=false" >> $GITHUB_ENV
+
+      # to get the PR data that can be used for post merge actions
+      - uses: actions/github-script@v7
+        if: github.event_name == 'push'
+        id: get_pr_data
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            return (
+              await github.rest.repos.listPullRequestsAssociatedWithCommit({
+                commit_sha: context.sha,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+              })
+            ).data[0];
 
       - name: Setup environment variables (Push)
         if: github.event_name == 'push'
@@ -53,8 +69,9 @@ jobs:
           echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
           echo "pr_owner=$(jq --raw-output '.commits[0].author.username' $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-          echo "pr_number=Null" >> $GITHUB_ENV
+          echo 'pr_number=${{ fromJson(steps.get_pr_data.outputs.result).number }}' >> $GITHUB_ENV
           echo "pr_or_commit_description=$(jq --ascii-output .head_commit.message $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "post_merge_action=true" >> $GITHUB_ENV
 
       - name: Checkout opensearch-build repo
         uses: actions/checkout@v4