diff --git a/src/Lucene.Net/Search/ReferenceManager.cs b/src/Lucene.Net/Search/ReferenceManager.cs
index ee9c2f34b5..ccfb109f4e 100644
--- a/src/Lucene.Net/Search/ReferenceManager.cs
+++ b/src/Lucene.Net/Search/ReferenceManager.cs
@@ -56,7 +56,7 @@ protected G Current
 
         private readonly ReentrantLock refreshLock = new ReentrantLock();
 
-        private readonly ISet<ReferenceManager.IRefreshListener> refreshListeners = new ConcurrentHashSet<ReferenceManager.IRefreshListener>();
+        private readonly ConcurrentHashSet<ReferenceManager.IRefreshListener> refreshListeners = new ConcurrentHashSet<ReferenceManager.IRefreshListener>();
 
         private void EnsureOpen()
         {
@@ -367,7 +367,7 @@ public virtual void RemoveListener(ReferenceManager.IRefreshListener listener)
             {
                 throw new ArgumentNullException(nameof(listener), "Listener cannot be null"); // LUCENENET specific - changed from IllegalArgumentException to ArgumentNullException (.NET convention)
             }
-            refreshListeners.Remove(listener);
+            refreshListeners.TryRemove(listener);
         }
     }
 
@@ -396,4 +396,4 @@ public interface IRefreshListener
             void AfterRefresh(bool didRefresh);
         }
     }
-}
\ No newline at end of file
+}