Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix: Resolve error TypeError "unhashable type list" (2nd pass) #251

Merged
merged 8 commits into from
May 21, 2024
Merged

Fix: Resolve error TypeError "unhashable type list" (2nd pass) #251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
64be057
chore: TDD: add failing tests
aaronsteers May 20, 2024
c79e8dd
chore: add more exception handling
aaronsteers May 20, 2024
3d2d142
set resolved type to varchar instead of object
aaronsteers May 21, 2024
5707380
fix tests, add comments on future work
aaronsteers May 21, 2024
afa4d74
add mising catalog declaration
aaronsteers May 21, 2024
e81072b
fix expected value
aaronsteers May 21, 2024
1f2d9e1
remove redundant/stale test
aaronsteers May 21, 2024
59174b6
remove stale redundant test
aaronsteers May 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions airbyte/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,13 @@ def _get_airbyte_type( # noqa: PLR0911 # Too many return statements
if json_schema_format == "time":
return "time_without_timezone", None

if json_schema_type in {"string", "number", "boolean", "integer"}:
return cast(str, json_schema_type), None
if isinstance(json_schema_type, str) and json_schema_type in {
"string",
"number",
"boolean",
"integer",
}:
return json_schema_type, None

if json_schema_type == "object":
return "object", None
Expand Down
25 changes: 23 additions & 2 deletions tests/integration_tests/fixtures/source-test/source_test/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@
"properties": {
"Column1": {"type": "string"},
"Column2": {"type": "number"},
"sometimes_object": {
"type": [
"null",
"string",
"object",
],
"properties": {
"nested_column": {"type": "string"},
},
},
},
},
},
Expand Down Expand Up @@ -87,15 +97,26 @@
sample_record1_stream1 = {
"type": "RECORD",
"record": {
"data": {"Column1": "value1", "Column2": 1},
"data": {
"Column1": "value1",
"Column2": 1,
# TODO: Output this as an object instead of a string
# Breaks tests.
# https://github.com/airbytehq/PyAirbyte/issues/253
"sometimes_object": '{"nested_column": "nested_value"}',
},
"stream": "stream1",
"emitted_at": 1704067200,
},
}
sample_record2_stream1 = {
"type": "RECORD",
"record": {
"data": {"Column1": "value2", "Column2": 2},
"data": {
"Column1": "value2",
"Column2": 2,
"sometimes_object": "string_value",
},
"stream": "stream1",
"emitted_at": 1704067200,
},
Expand Down
36 changes: 10 additions & 26 deletions tests/integration_tests/test_source_test_fixture.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,16 @@ def source_test(source_test_env) -> ab.Source:
def expected_test_stream_data() -> dict[str, list[dict[str, str | int]]]:
return {
"stream1": [
{"column1": "value1", "column2": 1},
{"column1": "value2", "column2": 2},
{
"column1": "value1",
"column2": 1,
"sometimes_object": '{"nested_column": "nested_value"}',
},
{
"column1": "value2",
"column2": 2,
"sometimes_object": "string_value",
},
],
"stream2": [
{
Expand Down Expand Up @@ -322,27 +330,12 @@ def test_dataset_list_and_len(expected_test_stream_data):
lazy_dataset_list = list(lazy_dataset)
# Make sure counts are correct
assert len(list(lazy_dataset_list)) == 2
# Make sure records are correct
assert list(pop_internal_columns_from_dataset(lazy_dataset_list)) == [
{"column1": "value1", "column2": 1},
{"column1": "value2", "column2": 2},
]

# Test the cached dataset implementation
result: ReadResult = source.read(ab.new_local_cache())
stream_1 = result["stream1"]
assert len(stream_1) == 2
assert len(list(stream_1)) == 2
# Make sure we can iterate over the stream after calling len
assert list(pop_internal_columns_from_dataset(stream_1)) == [
{"column1": "value1", "column2": 1},
{"column1": "value2", "column2": 2},
]
# Make sure we can iterate over the stream a second time
assert list(pop_internal_columns_from_dataset(stream_1)) == [
{"column1": "value1", "column2": 1},
{"column1": "value2", "column2": 2},
]

assert isinstance(result, Mapping)
assert "stream1" in result
Expand Down Expand Up @@ -809,15 +802,6 @@ def test_sync_limited_streams(expected_test_stream_data):
)


def test_read_stream():
source = ab.get_source("source-test", config={"apiKey": "test"})

assert pop_internal_columns_from_dataset(source.get_records("stream1")) == [
{"column1": "value1", "column2": 1},
{"column1": "value2", "column2": 2},
]


def test_read_stream_nonexisting():
source = ab.get_source("source-test", config={"apiKey": "test"})

Expand Down
60 changes: 41 additions & 19 deletions tests/unit_tests/test_type_translation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
from __future__ import annotations

import pytest
from airbyte.types import SQLTypeConversionError, SQLTypeConverter, _get_airbyte_type
from sqlalchemy import types
from airbyte.types import SQLTypeConverter, _get_airbyte_type


@pytest.mark.parametrize(
Expand Down Expand Up @@ -54,6 +54,12 @@
({"type": ["null", "array"], "items": {"type": "object"}}, types.JSON),
({"type": "object", "properties": {}}, types.JSON),
({"type": ["null", "object"], "properties": {}}, types.JSON),
(
{"type": ["null", "string", "object"], "properties": {}},
# TODO: Consider migrating to object-type handling instead of string
# https://github.com/airbytehq/PyAirbyte/issues/253
types.VARCHAR,
),
# Malformed JSON schema seen in the wild:
({"type": "array", "items": {}}, types.JSON),
({"type": ["null", "array"], "items": {"items": {}}}, types.JSON),
Expand All @@ -66,22 +72,23 @@ def test_to_sql_type(json_schema_property_def, expected_sql_type):


@pytest.mark.parametrize(
"json_schema_property_def, expected_airbyte_type",
"json_schema_property_def, expected_airbyte_type, raises",
[
({"type": "string"}, "string"),
({"type": ["boolean", "null"]}, "boolean"),
({"type": ["null", "boolean"]}, "boolean"),
({"type": "string"}, "string"),
({"type": ["null", "string"]}, "string"),
({"type": "boolean"}, "boolean"),
({"type": "string", "format": "date"}, "date"),
({"type": "string"}, "string", None),
({"type": ["boolean", "null"]}, "boolean", None),
({"type": ["null", "boolean"]}, "boolean", None),
({"type": "string"}, "string", None),
({"type": ["null", "string"]}, "string", None),
({"type": "boolean"}, "boolean", None),
({"type": "string", "format": "date"}, "date", None),
(
{
"type": "string",
"format": "date-time",
"airbyte_type": "timestamp_without_timezone",
},
"timestamp_without_timezone",
None,
),
(
{
Expand All @@ -90,6 +97,7 @@ def test_to_sql_type(json_schema_property_def, expected_sql_type):
"airbyte_type": "timestamp_with_timezone",
},
"timestamp_with_timezone",
None,
),
(
{
Expand All @@ -98,26 +106,40 @@ def test_to_sql_type(json_schema_property_def, expected_sql_type):
"airbyte_type": "time_without_timezone",
},
"time_without_timezone",
None,
),
(
{"type": "string", "format": "time", "airbyte_type": "time_with_timezone"},
"time_with_timezone",
None,
),
({"type": "integer"}, "integer"),
({"type": "number", "airbyte_type": "integer"}, "integer"),
({"type": "number"}, "number"),
({"type": "integer"}, "integer", None),
({"type": "number", "airbyte_type": "integer"}, "integer", None),
({"type": "number"}, "number", None),
# Array type:
({"type": "array"}, "array"),
({"type": "array", "items": {"type": "object"}}, "array"),
({"type": ["null", "array"], "items": {"type": "object"}}, "array"),
({"type": "array"}, "array", None),
({"type": "array", "items": {"type": "object"}}, "array", None),
({"type": ["null", "array"], "items": {"type": "object"}}, "array", None),
# Object type:
({"type": "object"}, "object"),
({"type": "object"}, "object", None),
({"type": ["null", "object", "string"]}, None, SQLTypeConversionError),
({"type": ["not-a-type"]}, None, SQLTypeConversionError),
({"tyyyype": ["not-a-type"]}, None, SQLTypeConversionError),
# Malformed JSON schema seen in the wild:
({"type": "array", "items": {"items": {}}}, "array"),
({"type": ["null", "array"], "items": {"items": {}}}, "array"),
({"type": "array", "items": {"items": {}}}, "array", None),
({"type": ["null", "array"], "items": {"items": {}}}, "array", None),
],
)
def test_to_airbyte_type(json_schema_property_def, expected_airbyte_type):
def test_to_airbyte_type(
json_schema_property_def,
expected_airbyte_type: str,
raises: type[Exception] | None,
):
if raises:
with pytest.raises(raises):
_get_airbyte_type(json_schema_property_def)
return

airbyte_type, _ = _get_airbyte_type(json_schema_property_def)
assert airbyte_type == expected_airbyte_type

Expand Down