diff --git a/src/litdata/streaming/config.py b/src/litdata/streaming/config.py index 985ddeee..1808a108 100644 --- a/src/litdata/streaming/config.py +++ b/src/litdata/streaming/config.py @@ -226,7 +226,10 @@ def __getitem__(self, index: ChunkedIndex) -> Tuple[str, int, int]: begin = self._intervals[index.chunk_index][0] - filesize_bytes = (1 + chunk["chunk_size"]) * 4 + chunk["chunk_bytes"] + filesize_bytes = chunk["chunk_bytes"] + if self._config.get("encryption") is None and self._compressor is None: + filesize_bytes += (1 + chunk["chunk_size"]) * 4 + return local_chunkpath, begin, filesize_bytes def _get_chunk_index_from_filename(self, chunk_filename: str) -> int: