Skip to content

Commit

Permalink
clarifies that currently no support for nested json paths
Browse files Browse the repository at this point in the history
  • Loading branch information
willi-mueller committed Jul 12, 2024
1 parent 93c5af5 commit 858ec47
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions tests/extract/test_incremental.py
Original file line number Diff line number Diff line change
Expand Up @@ -698,14 +698,13 @@ def some_data(created_at=dlt.sources.incremental("created_at", on_cursor_value_n
assert s["last_value"] == 2


# @pytest.mark.parametrize("item_type", ALL_TEST_DATA_ITEM_FORMATS)
def test_cursor_path_none_can_raise_on_none() -> None:
data = [
# No nested json path support for pandas and arrow. See test_nested_cursor_path_arrow_fails
source_items = [
{"id": 1, "created_at": 1},
{"id": 2, "created_at": None},
{"id": 3, "created_at": 2},
]
source_items = data_to_item_format("object", data)

@dlt.resource
def some_data(created_at=dlt.sources.incremental("created_at", on_cursor_value_none="raise")):
Expand All @@ -717,13 +716,15 @@ def some_data(created_at=dlt.sources.incremental("created_at", on_cursor_value_n

# same thing when run in pipeline
with pytest.raises(PipelineStepFailed) as pip_ex:
dlt.run(some_data(), destination="dummy")
p = dlt.pipeline(pipeline_name=uniq_id())
p.extract(some_data())

assert isinstance(pip_ex.value.__context__, IncrementalCursorPathHasValueNone)
assert pip_ex.value.__context__.json_path == "created_at"


def test_cursor_path_none_nested_can_raise_on_none_1() -> None:
# No pandas and arrow. See test_nested_cursor_path_arrow_fails
# No nested json path support for pandas and arrow. See test_nested_cursor_path_arrow_fails
@dlt.resource
def some_data(
created_at=dlt.sources.incremental("data.items[0].created_at", on_cursor_value_none="raise")
Expand All @@ -749,6 +750,7 @@ def some_data(


def test_cursor_path_none_nested_can_include_on_none_1() -> None:
# No nested json path support for pandas and arrow. See test_nested_cursor_path_arrow_fails
@dlt.resource
def some_data(
created_at=dlt.sources.incremental(
Expand All @@ -762,6 +764,7 @@ def some_data(


def test_cursor_path_none_nested_can_include_on_none_2() -> None:
# No nested json path support for pandas and arrow. See test_nested_cursor_path_arrow_fails
@dlt.resource
def some_data(
created_at=dlt.sources.incremental(
Expand Down

0 comments on commit 858ec47

Please sign in to comment.