From 3f473d58e3dd04f75886e86cbbc46da7359b6a87 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Sat, 23 Jul 2022 10:00:14 -0700 Subject: [PATCH 1/5] [Segment Replication] Add snapshots tests with segment replication enabled Signed-off-by: Suraj Singh --- .../SegmentReplicationSnapshotIT.java | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java diff --git a/server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java new file mode 100644 index 0000000000000..d820e2c227f02 --- /dev/null +++ b/server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -0,0 +1,239 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import org.junit.BeforeClass; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.rest.RestStatus; +import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SegmentReplicationSnapshotIT extends AbstractSnapshotIntegTestCase { + private static final String INDEX_NAME = "test-segrep-idx"; + private static final String RESTORED_INDEX_NAME = INDEX_NAME + "-restored"; + private static final int SHARD_COUNT = 1; + private static final int REPLICA_COUNT = 1; + private static final int DOC_COUNT = 1010; + + private static final String REPOSITORY_NAME = "test-segrep-repo"; + private static final String SNAPSHOT_NAME = "test-segrep-snapshot"; + + @BeforeClass + public static void assumeFeatureFlag() { + assumeTrue("Segment replication Feature flag is enabled", Boolean.parseBoolean(System.getProperty(FeatureFlags.REPLICATION_TYPE))); + } + + public Settings segRepEnableIndexSettings() { + return getShardSettings() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + } + + public Settings docRepEnableIndexSettings() { + return getShardSettings() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .build(); + } + + public Settings.Builder getShardSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, SHARD_COUNT) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT); + } + + @Override + protected boolean addMockInternalEngine() { + return false; + } + + public void ingestData(int docCount, String indexName) throws Exception { + try( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random()) + ) { + indexer.start(docCount); + waitForDocs(docCount, indexer); + refresh(indexName); + } + } + + public void startClusterWithSettings(Settings indexSettings) throws Exception { + // Create 2 node cluster + final String nodeA = internalCluster().startNode(); + final String nodeB = internalCluster().startNode(); + createIndex(INDEX_NAME, indexSettings); + ensureGreen(INDEX_NAME); + // Ingest data + ingestData(DOC_COUNT, INDEX_NAME); + } + + public void createSnapshot() { + // Snapshot declaration + Path absolutePath = randomRepoPath().toAbsolutePath(); + logger.info("Path [{}]", absolutePath); + // Create snapshot + createRepository(REPOSITORY_NAME, "fs", absolutePath); + logger.info("Perform snapshot"); + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME) + .setWaitForCompletion(true) + .setIndices(INDEX_NAME) + .get(); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + } + + public RestoreSnapshotResponse restoreSnapshotWithSettings(Settings indexSettings) { + RestoreSnapshotRequestBuilder builder = client().admin() + .cluster() + .prepareRestoreSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME) + .setWaitForCompletion(false) + .setRenamePattern(INDEX_NAME) + .setRenameReplacement(RESTORED_INDEX_NAME); + if (indexSettings != null) { + builder.setIndexSettings(indexSettings); + } + return builder.get(); + } + + public void testRestoreOnSegRep() throws Exception { + startClusterWithSettings(segRepEnableIndexSettings()); + createSnapshot(); + // Delete index + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)); + Thread.sleep(5000); + assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); + + logger.info("Restore from snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(null); + + // Assertions + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + logger.info("Ensure cluster is green"); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + public void testRestoreOnSegRepDuringIngestion() throws Exception { + startClusterWithSettings(segRepEnableIndexSettings()); + createSnapshot(); + // Delete index + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)); + Thread.sleep(5000); + assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); + + logger.info("Restore from snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(null); + + // Assertions + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + logger.info("Ensure cluster is green"); + ingestData(5000, RESTORED_INDEX_NAME); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + public void testSnapshotOnDocRep_RestoreOnSegRep() throws Exception { + startClusterWithSettings(docRepEnableIndexSettings()); + createSnapshot(); + // Delete index + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)); + Thread.sleep(5000); + + logger.info("Restore from snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(segRepEnableIndexSettings()); + + // Assertions + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + logger.info("Ensure cluster is green"); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); + + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + public void testSnapshotOnSegRep_RestoreOnDocRep() throws Exception { + startClusterWithSettings(segRepEnableIndexSettings()); + createSnapshot(); + // Delete index + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)); + Thread.sleep(5000); + + logger.info("Restore from snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(docRepEnableIndexSettings()); + + // Assertions + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + logger.info("Wait for green index health"); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "DOCUMENT"); + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + public void testSnapshotOnDocRep_RestoreOnDocRep() throws Exception { + startClusterWithSettings(docRepEnableIndexSettings()); + createSnapshot(); + // Delete index + client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)); + Thread.sleep(5000); + + logger.info("Restore from snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(docRepEnableIndexSettings()); + + // Assertions + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + logger.info("Ensure cluster is green"); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "DOCUMENT"); + + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } +} From 6fda602f6bc9e2e3ca238483ce25e329dcc203e4 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Sat, 23 Jul 2022 10:41:57 -0700 Subject: [PATCH 2/5] Fix spotless failures Signed-off-by: Suraj Singh --- .../SegmentReplicationSnapshotIT.java | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) rename server/src/{test => internalClusterTest}/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java (88%) diff --git a/server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java similarity index 88% rename from server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java rename to server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java index d820e2c227f02..465718878ba6e 100644 --- a/server/src/test/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -20,7 +20,6 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.rest.RestStatus; @@ -49,15 +48,11 @@ public static void assumeFeatureFlag() { } public Settings segRepEnableIndexSettings() { - return getShardSettings() - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .build(); + return getShardSettings().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build(); } public Settings docRepEnableIndexSettings() { - return getShardSettings() - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) - .build(); + return getShardSettings().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT).build(); } public Settings.Builder getShardSettings() { @@ -73,7 +68,7 @@ protected boolean addMockInternalEngine() { } public void ingestData(int docCount, String indexName) throws Exception { - try( + try ( BackgroundIndexer indexer = new BackgroundIndexer( INDEX_NAME, "_doc", @@ -81,7 +76,8 @@ public void ingestData(int docCount, String indexName) throws Exception { -1, RandomizedTest.scaledRandomIntBetween(2, 5), false, - random()) + random() + ) ) { indexer.start(docCount); waitForDocs(docCount, indexer); @@ -147,7 +143,10 @@ public void testRestoreOnSegRep() throws Exception { assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); logger.info("Ensure cluster is green"); ensureGreen(RESTORED_INDEX_NAME); - GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)) + .get(); assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); assertHitCount(resp, DOC_COUNT); @@ -169,7 +168,10 @@ public void testRestoreOnSegRepDuringIngestion() throws Exception { logger.info("Ensure cluster is green"); ingestData(5000, RESTORED_INDEX_NAME); ensureGreen(RESTORED_INDEX_NAME); - GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)) + .get(); assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); assertHitCount(resp, DOC_COUNT); @@ -189,7 +191,10 @@ public void testSnapshotOnDocRep_RestoreOnSegRep() throws Exception { assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); logger.info("Ensure cluster is green"); ensureGreen(RESTORED_INDEX_NAME); - GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)) + .get(); assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); @@ -210,7 +215,10 @@ public void testSnapshotOnSegRep_RestoreOnDocRep() throws Exception { assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); logger.info("Wait for green index health"); ensureGreen(RESTORED_INDEX_NAME); - GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)) + .get(); assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "DOCUMENT"); SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); assertHitCount(resp, DOC_COUNT); @@ -230,7 +238,10 @@ public void testSnapshotOnDocRep_RestoreOnDocRep() throws Exception { assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); logger.info("Ensure cluster is green"); ensureGreen(RESTORED_INDEX_NAME); - GetSettingsResponse settingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)).get(); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)) + .get(); assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "DOCUMENT"); SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); From 988d007a95c3a9d31d355ec40c60694b5525f2a3 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 16 Sep 2022 11:38:21 -0700 Subject: [PATCH 3/5] Add changelog entry, address review comments, add failover test Signed-off-by: Suraj Singh --- CHANGELOG.md | 3 +- .../SegmentReplicationSnapshotIT.java | 91 ++++++++++++++----- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7839d43209a71..bbe79abf3e38a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - 2.3.0 release notes ([#4457](https://github.com/opensearch-project/OpenSearch/pull/4457)) - Added missing javadocs for `:distribution:tools` modules ([#4483](https://github.com/opensearch-project/OpenSearch/pull/4483)) - Add BWC version 2.3.1 ([#4513](https://github.com/opensearch-project/OpenSearch/pull/4513)) +- [Segment Replication] Add snapshot and restore tests for segment replication feature ([#3993](https://github.com/opensearch-project/OpenSearch/pull/3993)) ### Dependencies - Bumps `reactive-streams` from 1.0.3 to 1.0.4 @@ -91,4 +92,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) [Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD -[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x \ No newline at end of file +[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java index 465718878ba6e..193b28af6d0de 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -24,11 +24,15 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.rest.RestStatus; import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) @@ -44,7 +48,7 @@ public class SegmentReplicationSnapshotIT extends AbstractSnapshotIntegTestCase @BeforeClass public static void assumeFeatureFlag() { - assumeTrue("Segment replication Feature flag is enabled", Boolean.parseBoolean(System.getProperty(FeatureFlags.REPLICATION_TYPE))); +// assumeTrue("Segment replication Feature flag is enabled", Boolean.parseBoolean(System.getProperty(FeatureFlags.REPLICATION_TYPE))); } public Settings segRepEnableIndexSettings() { @@ -62,6 +66,14 @@ public Settings.Builder getShardSettings() { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT); } + public Settings restoreIndexSegRepSettings() { + return Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build(); + } + + public Settings restoreIndexDocRepSettings() { + return Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT).build(); + } + @Override protected boolean addMockInternalEngine() { return false; @@ -70,7 +82,7 @@ protected boolean addMockInternalEngine() { public void ingestData(int docCount, String indexName) throws Exception { try ( BackgroundIndexer indexer = new BackgroundIndexer( - INDEX_NAME, + indexName, "_doc", client(), -1, @@ -85,14 +97,20 @@ public void ingestData(int docCount, String indexName) throws Exception { } } - public void startClusterWithSettings(Settings indexSettings) throws Exception { - // Create 2 node cluster - final String nodeA = internalCluster().startNode(); - final String nodeB = internalCluster().startNode(); + // Start cluster with provided settings and return the node names as list + public List startClusterWithSettings(Settings indexSettings, int replicaCount) throws Exception { + // Start primary + final String primaryNode = internalCluster().startNode(); + List nodeNames = new ArrayList<>(); + nodeNames.add(primaryNode); + for(int i=0; i nodeNames = startClusterWithSettings(segRepEnableIndexSettings(), 1); + final String primaryNode = nodeNames.get(0); + createSnapshot(); + // Delete index + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); + assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); + + // stop the primary node so that restoration happens on replica node + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(null); + + // Assertions + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + logger.info("Ensure cluster is green"); + internalCluster().startNode(); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME)) + .get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, "index.replication.type"), "SEGMENT"); + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } } From 437aa13e3aac4f781297826cd2ae73b29e70f5e2 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 16 Sep 2022 11:41:45 -0700 Subject: [PATCH 4/5] Fix spotless failures Signed-off-by: Suraj Singh --- .../opensearch/snapshots/SegmentReplicationSnapshotIT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java index 193b28af6d0de..322a0085692e0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -48,7 +48,7 @@ public class SegmentReplicationSnapshotIT extends AbstractSnapshotIntegTestCase @BeforeClass public static void assumeFeatureFlag() { -// assumeTrue("Segment replication Feature flag is enabled", Boolean.parseBoolean(System.getProperty(FeatureFlags.REPLICATION_TYPE))); + assumeTrue("Segment replication Feature flag is enabled", Boolean.parseBoolean(System.getProperty(FeatureFlags.REPLICATION_TYPE))); } public Settings segRepEnableIndexSettings() { @@ -103,7 +103,7 @@ public List startClusterWithSettings(Settings indexSettings, int replica final String primaryNode = internalCluster().startNode(); List nodeNames = new ArrayList<>(); nodeNames.add(primaryNode); - for(int i=0; i Date: Fri, 16 Sep 2022 11:55:25 -0700 Subject: [PATCH 5/5] Address review comments 2 Signed-off-by: Suraj Singh --- .../snapshots/SegmentReplicationSnapshotIT.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java index 322a0085692e0..d92f2af3f4bfd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -116,10 +116,8 @@ public List startClusterWithSettings(Settings indexSettings, int replica public void createSnapshot() { // Snapshot declaration Path absolutePath = randomRepoPath().toAbsolutePath(); - logger.info("Path [{}]", absolutePath); // Create snapshot createRepository(REPOSITORY_NAME, "fs", absolutePath); - logger.info("Perform snapshot"); CreateSnapshotResponse createSnapshotResponse = client().admin() .cluster() .prepareCreateSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME) @@ -154,12 +152,10 @@ public void testRestoreOnSegRep() throws Exception { assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); - logger.info("Restore from snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(null); // Assertions assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - logger.info("Ensure cluster is green"); ensureGreen(RESTORED_INDEX_NAME); GetSettingsResponse settingsResponse = client().admin() .indices() @@ -177,12 +173,10 @@ public void testSnapshotOnSegRep_RestoreOnSegRepDuringIngestion() throws Excepti assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); - logger.info("Restore from snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(null); // Assertions assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - logger.info("Ensure cluster is green"); ingestData(5000, RESTORED_INDEX_NAME); ensureGreen(RESTORED_INDEX_NAME); GetSettingsResponse settingsResponse = client().admin() @@ -200,12 +194,10 @@ public void testSnapshotOnDocRep_RestoreOnSegRep() throws Exception { // Delete index assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); - logger.info("Restore from snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(restoreIndexSegRepSettings()); // Assertions assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - logger.info("Ensure cluster is green"); ensureGreen(RESTORED_INDEX_NAME); GetSettingsResponse settingsResponse = client().admin() .indices() @@ -224,12 +216,10 @@ public void testSnapshotOnSegRep_RestoreOnDocRep() throws Exception { // Delete index assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); - logger.info("Restore from snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(restoreIndexDocRepSettings()); // Assertions assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - logger.info("Wait for green index health"); ensureGreen(RESTORED_INDEX_NAME); GetSettingsResponse settingsResponse = client().admin() .indices() @@ -246,12 +236,10 @@ public void testSnapshotOnDocRep_RestoreOnDocRep() throws Exception { // Delete index assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); - logger.info("Restore from snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(restoreIndexDocRepSettings()); // Assertions assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - logger.info("Ensure cluster is green"); ensureGreen(RESTORED_INDEX_NAME); GetSettingsResponse settingsResponse = client().admin() .indices() @@ -278,7 +266,6 @@ public void testRestoreOnReplicaNode() throws Exception { // Assertions assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - logger.info("Ensure cluster is green"); internalCluster().startNode(); ensureGreen(RESTORED_INDEX_NAME); GetSettingsResponse settingsResponse = client().admin()