pass storage options to s5cmd

src/litdata/streaming/downloader.py

@@ -68,19 +68,21 @@ def download_file(self, remote_filepath: str, local_filepath: str) -> None:
             local_filepath + ".lock", timeout=3 if obj.path.endswith(_INDEX_FILENAME) else 0
         ):
             if self._s5cmd_available:
+                env = os.environ.copy()
+                if self._storage_options:
+                    env.update(self._storage_options)
                 proc = subprocess.Popen(
                     f"s5cmd cp {remote_filepath} {local_filepath}",
                     shell=True,
                     stdout=subprocess.PIPE,
+                    env=env,
                 )
                 proc.wait()
             else:
                 from boto3.s3.transfer import TransferConfig
 
                 extra_args: Dict[str, Any] = {}
 
-                # try:
-                #     with FileLock(local_filepath + ".lock", timeout=1):
                 if not os.path.exists(local_filepath):
                     # Issue: https://github.com/boto/boto3/issues/3113
                     self._client.client.download_file(