feat(s3/retry_open): add comment

ToucanToco · Nov 3, 2021 · 67c2791 · 67c2791
1 parent a9b1214
commit 67c2791
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/peakina/io/s3/s3_utils.py b/peakina/io/s3/s3_utils.py
@@ -61,7 +61,11 @@ def _s3_open_file_with_retries(fs, path, retries):
             return file
         except Exception as ex:
             logger.warning(f'could not open {path}', ex)
+            # if the file has just been uploaded, then it might not be visible immediatly
+            # but the fail to open has been cached by s3fs
+            # so, we invalidate the cache
             fs.invalidate_cache(path)
+            # and we give some time to S3 to settle the file status
             sleep(1)