update

Lightning-AI · Oct 11, 2024 · 832ba55 · 832ba55
1 parent 3a832ae
commit 832ba55
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/src/litdata/streaming/config.py b/src/litdata/streaming/config.py
@@ -226,7 +226,10 @@ def __getitem__(self, index: ChunkedIndex) -> Tuple[str, int, int]:
 
         begin = self._intervals[index.chunk_index][0]
 
-        filesize_bytes = (1 + chunk["chunk_size"]) * 4 + chunk["chunk_bytes"]
+        filesize_bytes = chunk["chunk_bytes"]
+        if self._config.get("encryption") is None and self._compressor is None:
+            filesize_bytes += (1 + chunk["chunk_size"]) * 4
+
         return local_chunkpath, begin, filesize_bytes
 
     def _get_chunk_index_from_filename(self, chunk_filename: str) -> int: