Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix code formatting #2580

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion deeplake/api/tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1922,7 +1922,9 @@ def test_dataset_copy(
[
("local_ds_generator", "local_path", "hub_cloud_dev_token"),
pytest.param(
"s3_ds_generator", "s3_path", "hub_cloud_dev_token",
"s3_ds_generator",
"s3_path",
"hub_cloud_dev_token",
marks=pytest.mark.slow,
),
pytest.param(
Expand Down
2 changes: 1 addition & 1 deletion deeplake/core/transform/transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ def my_fn(sample_in: Any, samples_out, my_arg0, my_arg1=0):
index=index,
sample=sample,
samples_processed=samples_processed,
suggest=suggest,
suggest=suggest,
) from e
finally:
reload_and_rechunk(
Expand Down
37 changes: 25 additions & 12 deletions deeplake/enterprise/test_pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,11 @@ def test_setting_woker_init_function(local_auth_ds):
@pytest.mark.parametrize(
"ds",
[
pytest.param("hub_cloud_ds", marks=[pytest.mark.slow, pytest.mark.skip("Causing lockups")]),
"local_auth_ds"
pytest.param(
"hub_cloud_ds",
marks=[pytest.mark.slow, pytest.mark.skip("Causing lockups")],
),
"local_auth_ds",
],
indirect=True,
)
Expand Down Expand Up @@ -554,10 +557,13 @@ def test_rename(local_auth_ds):

@requires_torch
@requires_libdeeplake
@pytest.mark.parametrize("num_workers", [
0,
pytest.param(2, marks=pytest.mark.skip(reason="causing lockups")),
])
@pytest.mark.parametrize(
"num_workers",
[
0,
pytest.param(2, marks=pytest.mark.skip(reason="causing lockups")),
],
)
@pytest.mark.slow
@pytest.mark.flaky
def test_indexes(local_auth_ds, num_workers):
Expand All @@ -580,10 +586,13 @@ def test_indexes(local_auth_ds, num_workers):
@requires_torch
@requires_libdeeplake
@pytest.mark.slow
@pytest.mark.parametrize("num_workers", [
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups")),
])
@pytest.mark.parametrize(
"num_workers",
[
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups")),
],
)
@pytest.mark.flaky
def test_indexes_transform(local_auth_ds, num_workers):
shuffle = False
Expand Down Expand Up @@ -611,7 +620,9 @@ def test_indexes_transform(local_auth_ds, num_workers):

@requires_torch
@requires_libdeeplake
@pytest.mark.parametrize("num_workers", [0, pytest.param(2, marks=pytest.mark.skip("causing lockups"))])
@pytest.mark.parametrize(
"num_workers", [0, pytest.param(2, marks=pytest.mark.skip("causing lockups"))]
)
@pytest.mark.slow
@pytest.mark.flaky
def test_indexes_transform_dict(local_auth_ds, num_workers):
Expand Down Expand Up @@ -650,7 +661,9 @@ def test_indexes_transform_dict(local_auth_ds, num_workers):

@requires_torch
@requires_libdeeplake
@pytest.mark.parametrize("num_workers", [0, pytest.param(2, marks=pytest.mark.skip("causing lockups"))])
@pytest.mark.parametrize(
"num_workers", [0, pytest.param(2, marks=pytest.mark.skip("causing lockups"))]
)
@pytest.mark.slow
@pytest.mark.flaky
def test_indexes_tensors(local_auth_ds, num_workers):
Expand Down
71 changes: 27 additions & 44 deletions deeplake/enterprise/test_tensorflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,7 @@ def test_tensorflow_small(local_auth_ds):
ds.create_tensor("image2", max_chunk_size=TF_TESTS_MAX_CHUNK_SIZE)
ds.image2.extend(np.array([i * np.ones((12, 12)) for i in range(16)]))

if isinstance(
get_base_storage(ds.storage), (MemoryProvider, GCSProvider)
):
if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
with pytest.raises(ValueError):
dl = ds.dataloader()
return
Expand Down Expand Up @@ -130,9 +128,7 @@ def test_tensorflow_transform(local_auth_ds):
ds.create_tensor("image2", max_chunk_size=TF_TESTS_MAX_CHUNK_SIZE)
ds.image2.extend(np.array([i * np.ones((12, 12)) for i in range(16)]))

if isinstance(
get_base_storage(ds.storage), (MemoryProvider, GCSProvider)
):
if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
with pytest.raises(ValueError):
dl = ds.dataloader()
return
Expand Down Expand Up @@ -167,18 +163,12 @@ def test_tensorflow_transform_dict(local_auth_ds):
ds.create_tensor("image3", max_chunk_size=TF_TESTS_MAX_CHUNK_SIZE)
ds.image3.extend(np.array([i * np.ones((12, 12)) for i in range(16)]))

if isinstance(
get_base_storage(ds.storage), (MemoryProvider, GCSProvider)
):
if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
with pytest.raises(ValueError):
dl = ds.dataloader()
return

dl = (
ds.dataloader()
.transform({"image": double, "image2": None})
.tensorflow()
)
dl = ds.dataloader().transform({"image": double, "image2": None}).tensorflow()

assert len(dl.dataset) == 16

Expand Down Expand Up @@ -222,9 +212,7 @@ def test_tensorflow_with_compression(local_auth_ds: Dataset):
images.extend(np.ones((16, 12, 12, 3), dtype="uint8"))
labels.extend(np.ones((16, 1), dtype="uint32"))

if isinstance(
get_base_storage(ds.storage), (MemoryProvider, GCSProvider)
):
if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
with pytest.raises(ValueError):
dl = ds.dataloader()
return
Expand All @@ -250,19 +238,15 @@ def test_custom_tensor_order(local_auth_ds):
ds.create_tensor(t, max_chunk_size=TF_TESTS_MAX_CHUNK_SIZE)
ds[t].extend(np.random.random((3, 4, 5)))

if isinstance(
get_base_storage(ds.storage), (MemoryProvider, GCSProvider)
):
if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
with pytest.raises(ValueError):
dl = ds.dataloader()
return

with pytest.raises(TensorDoesNotExistError):
dl = ds.dataloader().tensorflow(tensors=["c", "d", "e"])

dl = ds.dataloader().tensorflow(
tensors=["c", "d", "a"], return_index=False
)
dl = ds.dataloader().tensorflow(tensors=["c", "d", "a"], return_index=False)

for i, batch in enumerate(dl):
c1, d1, a1 = batch
Expand Down Expand Up @@ -334,12 +318,8 @@ def test_groups(local_auth_ds, compressed_image_paths):
img1 = deeplake.read(compressed_image_paths["jpeg"][0])
img2 = deeplake.read(compressed_image_paths["png"][0])
with local_auth_ds as ds:
ds.create_tensor(
"images/jpegs/cats", htype="image", sample_compression="jpeg"
)
ds.create_tensor(
"images/pngs/flowers", htype="image", sample_compression="png"
)
ds.create_tensor("images/jpegs/cats", htype="image", sample_compression="jpeg")
ds.create_tensor("images/pngs/flowers", htype="image", sample_compression="png")
for _ in range(10):
ds.images.jpegs.cats.append(img1)
ds.images.pngs.flowers.append(img2)
Expand Down Expand Up @@ -559,10 +539,13 @@ def test_rename(local_auth_ds):

@requires_tensorflow
@requires_libdeeplake
@pytest.mark.parametrize("num_workers", [
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups")),
])
@pytest.mark.parametrize(
"num_workers",
[
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups")),
],
)
@pytest.mark.slow
@pytest.mark.flaky
def test_indexes(local_auth_ds, num_workers):
Expand All @@ -573,9 +556,7 @@ def test_indexes(local_auth_ds, num_workers):
ds.xyz.append(i * np.ones((2, 2)))

ptds = (
ds.dataloader()
.batch(4)
.tensorflow(num_workers=num_workers, return_index=True)
ds.dataloader().batch(4).tensorflow(num_workers=num_workers, return_index=True)
)
if shuffle:
ptds = ptds.shuffle()
Expand All @@ -588,10 +569,13 @@ def test_indexes(local_auth_ds, num_workers):

@requires_tensorflow
@requires_libdeeplake
@pytest.mark.parametrize("num_workers", [
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups")),
])
@pytest.mark.parametrize(
"num_workers",
[
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups")),
],
)
@pytest.mark.slow
@pytest.mark.flaky
def test_indexes_transform(local_auth_ds, num_workers):
Expand Down Expand Up @@ -660,10 +644,9 @@ def test_indexes_transform_dict(local_auth_ds, num_workers):

@requires_tensorflow
@requires_libdeeplake
@pytest.mark.parametrize("num_workers", [
0,
pytest.param(2, marks=pytest.mark.skip("causing lockups"))
])
@pytest.mark.parametrize(
"num_workers", [0, pytest.param(2, marks=pytest.mark.skip("causing lockups"))]
)
@pytest.mark.slow
@pytest.mark.flaky
def test_indexes_tensors(local_auth_ds, num_workers):
Expand Down
4 changes: 3 additions & 1 deletion deeplake/integrations/tests/test_pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,9 @@ def test_pytorch_collate(local_ds, shuffle, buffer_size):

@pytest.mark.slow
@requires_torch
@pytest.mark.parametrize("shuffle", [True, pytest.param(False, marks=pytest.mark.skip("causing lockups"))])
@pytest.mark.parametrize(
"shuffle", [True, pytest.param(False, marks=pytest.mark.skip("causing lockups"))]
)
@pytest.mark.flaky
def test_pytorch_transform_collate(local_ds, shuffle):
local_ds.create_tensor("a")
Expand Down
Loading