diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index d7037ab536dfa..acf3c5a3ee01d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -8,12 +8,17 @@ package org.opensearch.gateway.remote; +import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; +import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; +import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.coordination.CoordinationState; import org.opensearch.cluster.coordination.PersistedStateRegistry; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionResponse; import org.opensearch.gateway.GatewayMetaState; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; import org.opensearch.indices.recovery.RecoverySettings; @@ -27,9 +32,12 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Base64; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; @@ -161,8 +169,7 @@ public void testMasterReElectionUsesIncrementalUpload() throws IOException { ensureStableCluster(4); persistedStateRegistry = internalCluster().getClusterManagerNodeInstance(PersistedStateRegistry.class); - GatewayMetaState.RemotePersistedState persistedStateAfterElection = (GatewayMetaState.RemotePersistedState) persistedStateRegistry - .getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); + CoordinationState.PersistedState persistedStateAfterElection = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); ClusterMetadataManifest manifestAfterElection = persistedStateAfterElection.getLastAcceptedManifest(); // coordination metadata is updated, it will be unequal @@ -177,6 +184,41 @@ public void testMasterReElectionUsesIncrementalUpload() throws IOException { assertEquals(manifest.getIndicesRouting(), manifestAfterElection.getIndicesRouting()); } + + public void testVotingConfigAreCommitted() throws ExecutionException, InterruptedException { + prepareCluster(3, 2, INDEX_NAME, 1, 2); + ensureStableCluster(5); + ensureGreen(INDEX_NAME); + // add two new nodes to the cluster, to update the voting config + List newCMNodes = internalCluster().startClusterManagerOnlyNodes(2, Settings.EMPTY); + ensureStableCluster(7); + + internalCluster().getInstances(PersistedStateRegistry.class).forEach( + persistedStateRegistry -> { + CoordinationState.PersistedState localState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL); + CoordinationState.PersistedState remoteState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); + if (remoteState != null) { + assertEquals(localState.getLastAcceptedState().getLastCommittedConfiguration(), remoteState.getLastAcceptedState().getLastCommittedConfiguration()); + assertEquals(5, remoteState.getLastAcceptedState().getLastCommittedConfiguration().getNodeIds().size()); + } + } + ); + + // remove the newly added nodes from the voting config + client().admin().cluster().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(newCMNodes.get(0), newCMNodes.get(1))).get(); + + internalCluster().getInstances(PersistedStateRegistry.class).forEach( + persistedStateRegistry -> { + CoordinationState.PersistedState localState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL); + CoordinationState.PersistedState remoteState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); + if (remoteState != null) { + assertEquals(localState.getLastAcceptedState().getLastCommittedConfiguration(), remoteState.getLastAcceptedState().getLastCommittedConfiguration()); + assertEquals(3, remoteState.getLastAcceptedState().getLastCommittedConfiguration().getNodeIds().size()); + } + } + ); + } + private Map getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException { BlobPath metadataPath = repository.basePath() .add( diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index 8eadc57b69b9d..d7e62c8d78fea 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -797,18 +797,17 @@ public void markLastAcceptedStateAsCommitted() { assert lastAcceptedManifest != null : "Last accepted manifest is not present"; ClusterState clusterState = lastAcceptedState; boolean shouldCommitVotingConfig = shouldCommitVotingConfig(); - + Metadata.Builder metadataBuilder; + if (shouldCommitVotingConfig) { + metadataBuilder = commitVotingConfiguration(lastAcceptedState); + } else { + metadataBuilder = Metadata.builder(lastAcceptedState.metadata()); + } if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false && lastAcceptedState.metadata().clusterUUIDCommitted() == false) { - Metadata.Builder metadataBuilder; - if (shouldCommitVotingConfig) { - metadataBuilder = commitVotingConfiguration(lastAcceptedState); - } else { - metadataBuilder = Metadata.builder(lastAcceptedState.metadata()); - } metadataBuilder.clusterUUIDCommitted(true); - clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build(); } + clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build(); if (clusterState.getNodes().isLocalNodeElectedClusterManager()) { final RemoteClusterStateManifestInfo committedManifestDetails = remoteClusterStateService.markLastStateAsCommitted( clusterState,