Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Bump pydantic to ^2.4.2 [TCTC-7241] #1272

Merged
merged 36 commits into from
Dec 15, 2023
Merged

chore: Bump pydantic to ^2.4.2 [TCTC-7241] #1272

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
65a9774
chore: Bump pydantic to ^2.3.0
lukapeschke Sep 11, 2023
e178f74
chore: run bump-pydantic
lukapeschke Sep 11, 2023
51bb064
refactor: replace json_serializers
lukapeschke Sep 11, 2023
102eb89
fix: underscore_attrs_are_private
lukapeschke Sep 11, 2023
78e26f3
refactor: replace schema_extra
lukapeschke Sep 11, 2023
a0915c0
refactor validators with values
lukapeschke Sep 11, 2023
8143eda
chore: lint
lukapeschke Sep 11, 2023
22da7b4
fix: type annotation for oauth2_version
lukapeschke Sep 11, 2023
fdc3164
fix: replace root_validator with model_validator
lukapeschke Sep 11, 2023
6a19887
fix typing
lukapeschke Sep 11, 2023
a121226
refactor: Make data_source_model an attribute of __init_subclass__
lukapeschke Sep 11, 2023
0066455
update tests
lukapeschke Sep 11, 2023
9f0ce87
fix tests
lukapeschke Sep 12, 2023
0426ec6
Merge branch 'master' into upgrade-pydantic
lukapeschke Sep 12, 2023
8fa9d9a
blindly try to fix oracle tests
lukapeschke Sep 12, 2023
717f365
Merge branch 'master' into upgrade-pydantic
lukapeschke Sep 15, 2023
e47c477
adapt recently merged code
lukapeschke Sep 15, 2023
4e37a10
Merge branch 'master' into upgrade-pydantic
lukapeschke Sep 28, 2023
1744922
Update tests/google_sheets_2/test_google_sheets_2.py
lukapeschke Sep 29, 2023
d0dc261
bump pydandic ^2.4.2
lukapeschke Sep 29, 2023
3b3eb12
fix readme formatting
lukapeschke Sep 29, 2023
a9c0377
fix: remove merge conflict artifact
lukapeschke Sep 29, 2023
dcb3c28
chore: format
lukapeschke Sep 29, 2023
46bcf5b
Merge branch 'master' into upgrade-pydantic
lukapeschke Nov 22, 2023
778d2ac
Merge branch 'master' into upgrade-pydantic
lukapeschke Nov 22, 2023
eadde7a
chore: mypy
lukapeschke Nov 22, 2023
a3d0757
Merge branch 'master' into upgrade-pydantic
lukapeschke Nov 23, 2023
48a0560
Merge branch 'master' into upgrade-pydantic
lukapeschke Dec 4, 2023
f28474f
Merge branch 'master' into upgrade-pydantic
lukapeschke Dec 4, 2023
edd2858
fix: allow identifier to be None
lukapeschke Dec 5, 2023
2d6f547
fix: Add garbage code to mongo connector
lukapeschke Dec 6, 2023
6c44091
fix: do not use PlainJsonSecretStr for Oauth2
lukapeschke Dec 8, 2023
b96c321
fix horrible postgres typing
lukapeschke Dec 11, 2023
a7e52df
fix horrible http_api typing
lukapeschke Dec 11, 2023
a219852
fix horrible mysql typing
lukapeschke Dec 11, 2023
41b211d
Merge branch 'master' into upgrade-pydantic
lukapeschke Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 9 additions & 30 deletions tests/redshift/test_redshift.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from unittest.mock import Mock, patch

import pytest
from pytest_mock import MockerFixture
from redshift_connector.error import InterfaceError, OperationalError, ProgrammingError

from toucan_connectors.pagination import (
Expand Down Expand Up @@ -78,33 +79,10 @@ def redshift_datasource():
)


def test_config_schema_extra():
schema = {
'properties': {
'type': 'type_test',
'name': 'name_test',
'host': 'host_test',
'port': 0,
'cluster_identifier': 'cluster_identifier_test',
'db_user': 'db_user_test',
'connect_timeout': 'connect_timeout_test',
'authentication_method': 'authentication_method_test',
'user': 'user_test',
'password': 'password_test',
'default_database': 'dev',
'access_key_id': 'access_key_id_test',
'secret_access_key': 'secret_access_key_test',
'session_token': 'session_token_test',
'profile': 'profile_test',
'region': 'region_test',
'enable_tcp_keepalive': True,
}
}
RedshiftConnector.Config().schema_extra(schema)
assert schema['properties'] is not None
keys = list(schema['properties'].keys())
for i in range(len(keys)):
assert keys[i] == ORDERED_KEYS[i]
def test_model_json_schema():
schema = RedshiftConnector.model_json_schema()
for key, expected_key in zip(schema['properties'].keys(), ORDERED_KEYS):
assert key == expected_key


def test_redshiftdatasource_init_(redshift_datasource):
Expand All @@ -113,10 +91,11 @@ def test_redshiftdatasource_init_(redshift_datasource):
assert hasattr(ds, 'query_object')


@patch.object(RedshiftConnector, '_retrieve_tables')
def test_redshiftdatasource_get_form(redshift_connector, redshift_datasource):
def test_redshiftdatasource_get_form(
redshift_connector, redshift_datasource, mocker: MockerFixture
):
current_config = {'database': 'dev'}
redshift_connector._retrieve_tables.return_value = ['table1', 'table2', 'table3']
mocker.patch.object(RedshiftConnector, 'available_dbs', new=['one', 'two'])
result = redshift_datasource.get_form(redshift_connector, current_config)
assert result['properties']['parameters']['title'] == 'Parameters'
assert result['properties']['domain']['title'] == 'Domain'
Expand Down
2 changes: 1 addition & 1 deletion tests/soap/test_soap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def test_get_form(mocker, connector, create_datasource):
)
form = create_datasource.get_form(connector, {})
assert form['properties']['parameters']['description'] == 'Services documentation: <br> coucou'
assert form['definitions']['method']['enum'][0] == 'fake_func'
assert form['$defs']['method']['const'] == 'fake_func'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

J'aurais proposé de changer DEFAULT_REF_TEMPLATE pour changer le moins de tests possibles

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Je viens de tester avec ref_template='#/definitions/{model}', et les definitions restent malgré tout dans la clé $defs. Seule leur référence change. Exemple:

from enum import StrEnum
import json

from pydantic import BaseModel


class E(StrEnum):
    one = "one"
    two = "two"


class A(BaseModel):
    e: E


print(json.dumps(A.model_json_schema(), indent=2))
print(json.dumps(A.model_json_schema(ref_template="#/definitions/{model}"), indent=2))

Donne

{
  "$defs": {
    "E": {
      "enum": [
        "one",
        "two"
      ],
      "title": "E",
      "type": "string"
    }
  },
  "properties": {
    "e": {
      "$ref": "#/$defs/E"
    }
  },
  "required": [
    "e"
  ],
  "title": "A",
  "type": "object"
}
{
  "$defs": {
    "E": {
      "enum": [
        "one",
        "two"
      ],
      "title": "E",
      "type": "string"
    }
  },
  "properties": {
    "e": {
      "$ref": "#/definitions/E"
    }
  },
  "required": [
    "e"
  ],
  "title": "A",
  "type": "object"
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Je viens de tester avec ref_template='#/definitions/{model}', et les definitions restent malgré tout dans la clé $defs. Seule leur référence change. Exemple:

from enum import StrEnum
import json

from pydantic import BaseModel


class E(StrEnum):
    one = "one"
    two = "two"


class A(BaseModel):
    e: E


print(json.dumps(A.model_json_schema(), indent=2))
print(json.dumps(A.model_json_schema(ref_template="#/definitions/{model}"), indent=2))

Donne

{
  "$defs": {
    "E": {
      "enum": [
        "one",
        "two"
      ],
      "title": "E",
      "type": "string"
    }
  },
  "properties": {
    "e": {
      "$ref": "#/$defs/E"
    }
  },
  "required": [
    "e"
  ],
  "title": "A",
  "type": "object"
}
{
  "$defs": {
    "E": {
      "enum": [
        "one",
        "two"
      ],
      "title": "E",
      "type": "string"
    }
  },
  "properties": {
    "e": {
      "$ref": "#/definitions/E"
    }
  },
  "required": [
    "e"
  ],
  "title": "A",
  "type": "object"
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Je viens de tester avec ref_template='#/definitions/{model}', et les definitions restent malgré tout dans la clé $defs. Seule leur référence change. Exemple:

from enum import StrEnum
import json

from pydantic import BaseModel


class E(StrEnum):
    one = "one"
    two = "two"


class A(BaseModel):
    e: E


print(json.dumps(A.model_json_schema(), indent=2))
print(json.dumps(A.model_json_schema(ref_template="#/definitions/{model}"), indent=2))

Donne

{
  "$defs": {
    "E": {
      "enum": [
        "one",
        "two"
      ],
      "title": "E",
      "type": "string"
    }
  },
  "properties": {
    "e": {
      "$ref": "#/$defs/E"
    }
  },
  "required": [
    "e"
  ],
  "title": "A",
  "type": "object"
}
{
  "$defs": {
    "E": {
      "enum": [
        "one",
        "two"
      ],
      "title": "E",
      "type": "string"
    }
  },
  "properties": {
    "e": {
      "$ref": "#/definitions/E"
    }
  },
  "required": [
    "e"
  ],
  "title": "A",
  "type": "object"
}

Copy link
Member

@PrettyWood PrettyWood Sep 14, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

c'est pas un bug ? 🤔
A priori non



def test_create_client(mocker, connector):
Expand Down
88 changes: 35 additions & 53 deletions tests/test_connector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pandas as pd
import pytest
import tenacity as tny
from pydantic import create_model
from pydantic import ValidationError, create_model

from toucan_connectors.common import ConnectorStatus
from toucan_connectors.google_sheets_2.google_sheets_2_connector import GoogleSheets2Connector
Expand All @@ -29,9 +29,8 @@ class DataSource(ToucanDataSource):
parameters: dict = {}


class DataConnector(ToucanConnector):
type = 'MyDB'
data_source_model: DataSource
class DataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyDB'
a_parameter: str = ''

def _retrieve_data(self, data_source):
Expand All @@ -41,21 +40,18 @@ def _retrieve_data(self, data_source):
################################################
def test_missing_attributes():
# missing data_source_model
with pytest.raises(TypeError) as exc_info:
with pytest.raises(TypeError, match='data_source_model'):

class MissingDataConnector2(ToucanConnector):
type = 'MyDB'
type: str = 'MyDB'

def _retrieve_data(self, data_source):
pass

assert str(exc_info.value) == "MissingDataConnector2 has no 'data_source_model' attribute."


def test_no_get_df():
class BadDataConnector(ToucanConnector):
type = 'MyDB'
data_source_model = 'asd'
class BadDataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyDB'

with pytest.raises(TypeError):
BadDataConnector(name='my_name')
Expand All @@ -74,9 +70,8 @@ def test_validate():


def test_formated_engine_version():
class DataConnector(ToucanConnector, VersionableEngineConnector):
type = 'MyDB'
data_source_model: DataSource
class DataConnector(ToucanConnector, VersionableEngineConnector, data_source_model=DataSource):
type: str = 'MyDB'

def get_engine_version(self) -> tuple:
return super().get_engine_version()
Expand All @@ -99,9 +94,8 @@ def _retrieve_data(self, datasource):


def test_get_df_with_permissions():
class DataConnector(ToucanConnector):
type = 'MyDB'
data_source_model: DataSource
class DataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyDB'

def _retrieve_data(self, datasource):
return pd.DataFrame({'A': [1, 2]})
Expand All @@ -113,9 +107,8 @@ def _retrieve_data(self, datasource):


def test_get_slice():
class DataConnector(ToucanConnector):
type = 'MyDB'
data_source_model = 'asd'
class DataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyDB'

def _retrieve_data(self, datasource):
return pd.DataFrame({'A': [1, 2, 3, 4, 5]})
Expand Down Expand Up @@ -144,9 +137,8 @@ def _retrieve_data(self, datasource):


def test_explain():
class DataConnector(ToucanConnector):
type = 'MyDB'
data_source_model = 'asd'
class DataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyDB'

def _retrieve_data(self, datasource):
return pd.DataFrame()
Expand All @@ -166,7 +158,7 @@ def test_get_cache_key():
key = connector.get_cache_key(ds)
# We should get a deterministic identifier:
# /!\ the identifier will change if the model of the connector or the datasource changes
assert key == '10ae8360-5e30-319e-8cb8-a8a817826d12'
assert key == 'cc66ddcd-e717-381f-838e-f960e6cb410e'

ds.query = 'wow'
key2 = connector.get_cache_key(ds)
Expand Down Expand Up @@ -212,9 +204,8 @@ def test_get_cache_key_should_be_different_with_different_permissions():
assert key_a1 != key_a2


class UnreliableDataConnector(ToucanConnector):
type = 'MyUnreliableDB'
data_source_model: DataSource
class UnreliableDataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyUnreliableDB'

def _retrieve_data(self, data_source, logbook=[]):
if len(logbook) < 3:
Expand All @@ -231,9 +222,8 @@ def test_max_attempt_df():
assert result == 42


class CustomPolicyDataConnector(ToucanConnector):
type = 'MyUnreliableDB'
data_source_model: DataSource
class CustomPolicyDataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyUnreliableDB'

def _retrieve_data(self, data_source, logbook=[]):
if len(logbook) < 3:
Expand All @@ -253,9 +243,8 @@ def test_custom_max_attempt_df():
assert result['1'].values.tolist() == [42, 32]


class CustomRetryOnDataConnector(ToucanConnector):
type = 'MyUnreliableDB'
data_source_model: DataSource
class CustomRetryOnDataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyUnreliableDB'
_retry_on = (ValueError,)

def _retrieve_data(self, data_source, logbook=[]):
Expand All @@ -272,9 +261,8 @@ def test_custom_retry_on_df():
udc.get_df({})


class CustomNoRetryOnDataConnector(ToucanConnector):
type = 'MyUnreliableDB'
data_source_model: DataSource
class CustomNoRetryOnDataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyUnreliableDB'

@property
def retry_decorator(self):
Expand All @@ -297,56 +285,51 @@ def test_no_retry_on_df():
def test_strlist_to_enum_required():
"""It should be required by default"""
model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi']))
assert model.schema() == {
assert model.model_json_schema() == {
'title': 'Test',
'type': 'object',
'definitions': {
'$defs': {
'pokemon': {
'description': 'An enumeration.',
'enum': ['pika', 'bulbi'],
'title': 'pokemon',
'type': 'string',
}
},
'properties': {'pokemon': {'$ref': '#/definitions/pokemon'}},
'properties': {'pokemon': {'$ref': '#/$defs/pokemon'}},
'required': ['pokemon'],
}


def test_strlist_to_enum_default_value():
"""It should be possible to add a default value (not required)"""
model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'], 'pika'))
assert model.schema() == {
assert model.model_json_schema() == {
'title': 'Test',
'type': 'object',
'definitions': {
'$defs': {
'pokemon': {
'description': 'An enumeration.',
'enum': ['pika', 'bulbi'],
'title': 'pokemon',
'type': 'string',
}
},
'properties': {
'pokemon': {'allOf': [{'$ref': '#/definitions/pokemon'}], 'default': 'pika'}
},
'properties': {'pokemon': {'allOf': [{'$ref': '#/$defs/pokemon'}], 'default': 'pika'}},
}


def test_should_return_connector_config_form():
assert (
get_connector_secrets_form(GoogleSheets2Connector).secrets_schema
== OAuth2ConnectorConfig.schema()
== OAuth2ConnectorConfig.model_json_schema()
)
assert get_connector_secrets_form(MongoConnector) is None


def test_get_df_int_column(mocker):
"""The int column should be casted as str"""

class DataConnector(ToucanConnector):
type = 'MyDB'
data_source_model: DataSource
class DataConnector(ToucanConnector, data_source_model=DataSource):
type: str = 'MyDB'

def _retrieve_data(self, datasource):
return pd.DataFrame({0: [1, 2]})
Expand All @@ -356,9 +339,8 @@ def _retrieve_data(self, datasource):


def test_default_implementation_of_discoverable_connector():
class DataConnector(ToucanConnector, DiscoverableConnector):
type = 'MyDB'
data_source_model: DataSource
class DataConnector(ToucanConnector, DiscoverableConnector, data_source_model=DataSource):
type: str = 'MyDB'

def _retrieve_data(self, datasource):
return pd.DataFrame()
Expand Down