Skip to content

Commit

Permalink
Revert
Browse files Browse the repository at this point in the history
  • Loading branch information
Sultan Iman committed Feb 6, 2024
1 parent 8aea674 commit 226f4cd
Showing 1 changed file with 67 additions and 0 deletions.
67 changes: 67 additions & 0 deletions tests/load/pipeline/test_pydantic_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,70 @@ def src():
"data_type": "text",
"nullable": True,
}


def test_flattens_model_when_skip_complex_types_is_not_set():
class Parent(BaseModel):
child: Child
optional_parent_attribute: t.Optional[str] = None
data_dictionary: t.Dict[str, t.Any] = None
dlt_config: t.ClassVar[DltConfig] = {"skip_complex_types": False}

example_data = {
"optional_parent_attribute": None,
"data_dictionary": {
"child_attribute": "any string",
},
"child": {
"child_attribute": "any string",
"optional_child_attribute": None,
},
}

@dlt.resource
def res():
yield [example_data]

@dlt.source(max_table_nesting=1)
def src():
yield res()

p = dlt.pipeline("example", full_refresh=True, destination="duckdb")
p.run(src(), table_name="items", columns=Parent)

with p.sql_client() as client:
with client.execute_query("SELECT * FROM items") as cursor:
loaded_values = {
col[0]: val
for val, col in zip(cursor.fetchall()[0], cursor.description)
if col[0] not in ("_dlt_id", "_dlt_load_id")
}

assert loaded_values == {
"child": '{"child_attribute":"any string","optional_child_attribute":null}',
"optional_parent_attribute": None,
"data_dictionary": '{"child_attribute":"any string"}',
}

keys = p.default_schema.tables["items"]["columns"].keys()
assert keys == {
"child",
"optional_parent_attribute",
"data_dictionary",
"_dlt_load_id",
"_dlt_id",
}

columns = p.default_schema.tables["items"]["columns"]

assert columns["optional_parent_attribute"] == {
"name": "optional_parent_attribute",
"data_type": "text",
"nullable": True,
}

assert columns["data_dictionary"] == {
"name": "data_dictionary",
"data_type": "complex",
"nullable": False,
}

0 comments on commit 226f4cd

Please sign in to comment.