Make path tests Windows compatible (#1384)

* make path tests windows compatible --------- Co-authored-by: Jorrit Sandbrink <[email protected]>
dlt-hub · May 20, 2024 · 7a99633 · 7a99633
1 parent 4c6f928
commit 7a99633
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 25 deletions.
diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py
@@ -1,6 +1,7 @@
 import posixpath
 import os
 from unittest import mock
+from pathlib import Path
 
 import pytest
 
@@ -117,16 +118,18 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non
         client, _, root_path, load_id1 = load_info
         layout = client.config.layout
         # this path will be kept after replace
-        job_2_load_1_path = posixpath.join(
-            root_path,
-            create_path(
-                layout,
-                NORMALIZED_FILES[1],
-                client.schema.name,
-                load_id1,
-                load_package_timestamp=timestamp,
-                extra_placeholders=client.config.extra_placeholders,
-            ),
+        job_2_load_1_path = Path(
+            posixpath.join(
+                root_path,
+                create_path(
+                    layout,
+                    NORMALIZED_FILES[1],
+                    client.schema.name,
+                    load_id1,
+                    load_package_timestamp=timestamp,
+                    extra_placeholders=client.config.extra_placeholders,
+                ),
+            )
         )
 
         with perform_load(
@@ -135,16 +138,18 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non
             client, _, root_path, load_id2 = load_info
 
             # this one we expect to be replaced with
-            job_1_load_2_path = posixpath.join(
-                root_path,
-                create_path(
-                    layout,
-                    NORMALIZED_FILES[0],
-                    client.schema.name,
-                    load_id2,
-                    load_package_timestamp=timestamp,
-                    extra_placeholders=client.config.extra_placeholders,
-                ),
+            job_1_load_2_path = Path(
+                posixpath.join(
+                    root_path,
+                    create_path(
+                        layout,
+                        NORMALIZED_FILES[0],
+                        client.schema.name,
+                        load_id2,
+                        load_package_timestamp=timestamp,
+                        extra_placeholders=client.config.extra_placeholders,
+                    ),
+                )
             )
 
             # First file from load1 remains, second file is replaced by load2
@@ -159,7 +164,7 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non
                 for f in files:
                     if f == INIT_FILE_NAME:
                         continue
-                    paths.append(posixpath.join(basedir, f))
+                    paths.append(Path(posixpath.join(basedir, f)))
 
             ls = set(paths)
             assert ls == {job_2_load_1_path, job_1_load_2_path}
@@ -210,7 +215,7 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None
                 )
                 for job in jobs2
             ]
-            expected_files = sorted([posixpath.join(root_path, fn) for fn in expected_files])
+            expected_files = sorted([Path(posixpath.join(root_path, fn)) for fn in expected_files])  # type: ignore[misc]
 
             paths = []
             for basedir, _dirs, files in client.fs_client.walk(
@@ -222,5 +227,5 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None
                 for f in files:
                     if f == INIT_FILE_NAME:
                         continue
-                    paths.append(posixpath.join(basedir, f))
+                    paths.append(Path(posixpath.join(basedir, f)))
             assert list(sorted(paths)) == expected_files
diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py
@@ -301,7 +301,7 @@ def count(*args, **kwargs) -> Any:
 
         for file in files:
             if ".jsonl" in file:
-                expected_files.add(posixpath.join(basedir, file))
+                expected_files.add(Path(posixpath.join(basedir, file)))
 
     for load_package in load_info.load_packages:
         for load_info in load_package.jobs["completed_jobs"]:  # type: ignore[assignment]
@@ -321,7 +321,7 @@ def count(*args, **kwargs) -> Any:
             full_path = posixpath.join(client.dataset_path, path)  # type: ignore[attr-defined]
             assert client.fs_client.exists(full_path)  # type: ignore[attr-defined]
             if ".jsonl" in full_path:
-                known_files.add(full_path)
+                known_files.add(Path(full_path))
 
     assert expected_files == known_files
     assert known_files