[BUG] org.opensearch.index.shard.SegmentReplicationWithNodeToNodeIndexShardTests.testReplicaClosesWhileReplicating_AfterGetCheckpoint is flaky

[reta]

Describe the bug

The test case org.opensearch.index.shard.SegmentReplicationWithNodeToNodeIndexShardTests.testReplicaClosesWhileReplicating_AfterGetCheckpoint is flaky:

org.opensearch.index.shard.SegmentReplicationWithNodeToNodeIndexShardTests.testReplicaClosesWhileReplicating_AfterGetCheckpoint

java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there are still 1 open files: {_0.cfs=1}
	at __randomizedtesting.SeedInfo.seed([F01091562017B4DF:6D7E22AC42A8EDFB]:0)
	at org.apache.lucene.tests.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:876)
	at org.apache.lucene.store.FilterDirectory.close(FilterDirectory.java:111)
	at org.apache.lucene.store.FilterDirectory.close(FilterDirectory.java:111)
	at org.opensearch.index.store.Store$StoreDirectory.innerClose(Store.java:952)
	at org.opensearch.index.store.Store.closeInternal(Store.java:571)
	at org.opensearch.index.store.Store$1.closeInternal(Store.java:194)
	at org.opensearch.common.util.concurrent.AbstractRefCounted.decRef(AbstractRefCounted.java:78)
	at org.opensearch.index.store.Store.decRef(Store.java:546)
	at org.opensearch.index.store.Store.close(Store.java:553)
	at org.opensearch.common.util.io.IOUtils.close(IOUtils.java:89)
	at org.opensearch.common.util.io.IOUtils.close(IOUtils.java:131)
	at org.opensearch.common.util.io.IOUtils.close(IOUtils.java:81)
	at org.opensearch.index.shard.IndexShardTestCase.closeShard(IndexShardTestCase.java:965)
	at org.opensearch.index.shard.IndexShardTestCase.closeShards(IndexShardTestCase.java:972)
	at org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase.access$1000(OpenSearchIndexLevelReplicationTestCase.java:130)
	at org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase$ReplicationGroup.close(OpenSearchIndexLevelReplicationTestCase.java:610)
	at org.opensearch.index.shard.SegmentReplicationWithNodeToNodeIndexShardTests.testReplicaClosesWhileReplicating_AfterGetCheckpoint(SegmentReplicationWithNodeToNodeIndexShardTests.java:147)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.RuntimeException: unclosed IndexInput: _0.cfs
	at org.apache.lucene.tests.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:783)
	at org.apache.lucene.tests.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:835)
	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:101)
	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:101)
	at org.apache.lucene.codecs.lucene90.Lucene90CompoundReader.<init>(Lucene90CompoundReader.java:78)
	at org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat.getCompoundReader(Lucene90CompoundFormat.java:87)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:96)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:178)
	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:220)
	at org.apache.lucene.index.IndexWriter.lambda$getReader$0(IndexWriter.java:542)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:138)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:604)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:381)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:355)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:345)
	at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:170)
	at org.opensearch.index.engine.OpenSearchReaderManager.refreshIfNeeded(OpenSearchReaderManager.java:72)
	at org.opensearch.index.engine.OpenSearchReaderManager.refreshIfNeeded(OpenSearchReaderManager.java:52)
	at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:167)
	at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:240)
	at org.opensearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:433)
	at org.opensearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:413)
	at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:167)
	at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:240)
	at org.opensearch.index.engine.InternalEngine.refresh(InternalEngine.java:1771)
	at org.opensearch.index.engine.InternalEngine.refresh(InternalEngine.java:1748)
	at org.opensearch.index.shard.IndexShard.refresh(IndexShard.java:1341)
	at org.opensearch.index.shard.SegmentReplicationWithNodeToNodeIndexShardTests.testReplicaClosesWhileReplicating_AfterGetCheckpoint(SegmentReplicationWithNodeToNodeIndexShardTests.java:114)
	... 38 more

Related component

Storage:Remote

To Reproduce

./gradlew ':server:test' --tests "org.opensearch.index.shard.SegmentReplicationWithNodeToNodeIndexShardTests.testReplicaClosesWhileReplicating_AfterGetCheckpoint" -Dtests.seed=F01091562017B4DF

Expected behavior

The test must always pass

Additional Details

Plugins
Standard

Screenshots
If applicable, add screenshots to help explain your problem.

Host/Environment (please complete the following information):

• CI

Additional context

• https://build.ci.opensearch.org/job/gradle-check/33672/testReport/junit/org.opensearch.index.shard/SegmentReplicationWithNodeToNodeIndexShardTests/testReplicaClosesWhileReplicating_AfterGetCheckpoint/

[peternied]

[Triage - attendees 1 2 3 4 5 6 7 8]
@reta Thanks for filing