From 225814e492f2786d89fda36cd5662307a5f5b174 Mon Sep 17 00:00:00 2001 From: Playdev Date: Sat, 13 Jul 2024 19:47:22 +0900 Subject: [PATCH] docs: add `Specify cache directory` (#229) --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 78e91a5e..63b7c477 100644 --- a/README.md +++ b/README.md @@ -506,6 +506,24 @@ dataset = StreamingDataset(..., max_cache_size="10GB") +
+ ✅ Specify cache directory +  + +Specify the directory where cached files should be stored, ensuring efficient data retrieval and management. This is particularly useful for organizing your data storage and improving access times. + +```python +from litdata import StreamingDataset +from litdata.streaming.cache import Dir + +cache_dir = "/path/to/your/cache" +data_dir = "s3://my-bucket/my_optimized_dataset" + +dataset = StreamingDataset(input_dir=Dir(path=cache_dir, url=data_dir)) +``` + +
+
✅ Optimize loading on networked drives