Fix assertion failure while closing remoteStore

Signed-off-by: Sachin Kale <[email protected]>
opensearch-project · Oct 16, 2023 · d4c2011 · d4c2011
1 parent 01a6ffd
commit d4c2011
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -654,7 +654,15 @@ private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store
                 }
 
                 if (remoteStore != null && indexShard.isPrimaryMode() && deleted.get()) {
-                    remoteStore.close();
+                    // When an instance of Store is created, a shardlock is created which is released on closing the instance of store.
+                    // Currently, we create 2 instances of store for remote store backed indices: store and remoteStore.
+                    // As there can be only one shardlock acquired for a given shard, the lock is shared between store and remoteStore.
+                    // This creates an issue when we are deleting the index as it results in closing both store and remoteStore. At the time
+                    // of closure of second Store instance, we get the assertion error saying shard is not locked.
+                    // Ideally, we should be closing the remoteStore but until we work on CompositeStore
+                    // (https://github.com/opensearch-project/OpenSearch/issues/3719), we mitigate the test failures by
+                    // closing the remoteDirectory.
+                    indexShard.getRemoteDirectory().close();
                 }
 
             } catch (Exception e) {

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -1975,7 +1975,7 @@ public void close(String reason, boolean flushEngine, boolean deleted) throws IO
     /*
     ToDo : Fix this https://github.com/opensearch-project/OpenSearch/issues/8003
      */
-    private RemoteSegmentStoreDirectory getRemoteDirectory() {
+    public RemoteSegmentStoreDirectory getRemoteDirectory() {
         assert indexSettings.isRemoteStoreEnabled();
         assert remoteStore.directory() instanceof FilterDirectory : "Store.directory is not an instance of FilterDirectory";
         FilterDirectory remoteStoreDirectory = (FilterDirectory) remoteStore.directory();