From 832ba551fcb26af71853b6c078147bb2dc01ac18 Mon Sep 17 00:00:00 2001 From: tchaton Date: Fri, 11 Oct 2024 14:19:47 +0100 Subject: [PATCH] update --- src/litdata/streaming/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/litdata/streaming/config.py b/src/litdata/streaming/config.py index 985ddeee..1808a108 100644 --- a/src/litdata/streaming/config.py +++ b/src/litdata/streaming/config.py @@ -226,7 +226,10 @@ def __getitem__(self, index: ChunkedIndex) -> Tuple[str, int, int]: begin = self._intervals[index.chunk_index][0] - filesize_bytes = (1 + chunk["chunk_size"]) * 4 + chunk["chunk_bytes"] + filesize_bytes = chunk["chunk_bytes"] + if self._config.get("encryption") is None and self._compressor is None: + filesize_bytes += (1 + chunk["chunk_size"]) * 4 + return local_chunkpath, begin, filesize_bytes def _get_chunk_index_from_filename(self, chunk_filename: str) -> int: