diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
index 92436a09a4e7e..f1843ea3eef38 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
@@ -260,7 +260,7 @@ public long getRefreshSeqNoLag() {
     }
 
     public long getTimeMsLag() {
-        if (remoteRefreshTimeMs == localRefreshTimeMs) {
+        if (remoteRefreshTimeMs == localRefreshTimeMs || bytesLag == 0) {
             return 0;
         }
         return currentTimeMsUsingSystemNanos() - remoteRefreshStartTimeMs;