Resolve path resolution (#68)

Lightning-AI · Mar 8, 2024 · 6e6a629 · 6e6a629
1 parent 97257f3
commit 6e6a629
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/litdata/processing/data_processor.py b/litdata/processing/data_processor.py
@@ -349,9 +349,12 @@ def _is_path(input_dir: Optional[str], element: Any) -> bool:
 
         element = str(Path(element).absolute())
         if element.startswith(input_dir):
-            return True
+            # check whether the element has an extension.
+            if os.path.splitext(element)[1]:
+                return True
+            return os.path.isfile(element)
 
-    return os.path.exists(element)
+    return os.path.isfile(element)
 
 
 class BaseWorker: