feat: migration code for pydanticv2 support + lint

toucan_connectors/adobe_analytics/adobe_analytics_connector.py

@@ -3,7 +3,6 @@
 
 import pandas as pd
 from adobe_analytics import Client, ReportDefinition
-from pydantic import ConfigDict, __version__ as pydantic_version
 
 from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource
 
@@ -43,8 +42,6 @@ def report_definition(self):
             source=self.source,
         )
 
-_PYDANTIC_VERSION_ONE = pydantic_version.startswith("1.")
-
 
 class AdobeAnalyticsConnector(ToucanConnector):
     """
@@ -63,6 +60,3 @@ def _retrieve_data(self, data_source: AdobeAnalyticsDataSource) -> pd.DataFrame:
         df = suites[data_source.suite_id].download(data_source.report_definition)
         df['suite_id'] = data_source.suite_id
         return df
-
-    if not _PYDANTIC_VERSION_ONE:
-        model_config = ConfigDict(arbitrary_types_allowed=True)

toucan_connectors/anaplan/anaplan_connector.py

@@ -90,7 +90,7 @@ class AnaplanAuthError(AnaplanError):
 class AnaplanConnector(ToucanConnector):
     data_source_model: AnaplanDataSource
     username: str
-    password: Optional[SecretStr]
+    password: Optional[SecretStr] = None
 
     def _extract_json(self, resp: requests.Response) -> dict:
         if resp.status_code in (401, 403):

toucan_connectors/awsathena/awsathena_connector.py

@@ -4,7 +4,7 @@
 import boto3
 import pandas as pd
 from cached_property import cached_property_with_ttl
-from pydantic import Field, SecretStr, constr, create_model
+from pydantic import ConfigDict, Field, SecretStr, constr, create_model
 
 from toucan_connectors.common import ConnectorStatus, apply_query_parameters, sanitize_query
 from toucan_connectors.pagination import build_pagination_info
@@ -77,10 +77,11 @@ class AwsathenaConnector(ToucanConnector, DiscoverableConnector):
     aws_access_key_id: str = Field(..., description='Your AWS access key ID')
     aws_secret_access_key: SecretStr = Field(None, description='Your AWS secret key')
     region_name: str = Field(..., description='Your AWS region name')
-
-    class Config:
-        underscore_attrs_are_private = True
-        keep_untouched = (cached_property_with_ttl,)
+    # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = ConfigDict(
+        underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)
+    )
 
     def get_session(self) -> boto3.Session:
         return boto3.Session(

toucan_connectors/clickhouse/clickhouse_connector.py

@@ -24,6 +24,8 @@ class ClickhouseDataSource(ToucanDataSource):
         'your_table")',
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['ClickhouseDataSource']) -> None:

toucan_connectors/facebook_ads/facebook_ads_connector.py

@@ -78,7 +78,7 @@ def determine_query_params(self) -> Dict[str, str]:
 
 class FacebookAdsConnector(ToucanConnector):
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     _oauth_trigger = 'instance'
     data_source_model: FacebookAdsDataSource
     oauth2_version = Field('1', **{'ui.hidden': True})

toucan_connectors/github/github_connector.py

@@ -97,7 +97,7 @@ def get_form(cls, connector: 'GithubConnector', current_config, **kwargs):
 
 class GithubConnector(ToucanConnector):
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     data_source_model: GithubDataSource
     _oauth_trigger = 'instance'
     _oauth2_connector: OAuth2Connector = PrivateAttr()

toucan_connectors/google_adwords/google_adwords_connector.py

@@ -59,6 +59,8 @@ class GoogleAdwordsDataSource(ToucanDataSource):
         description='Max number of rows to extract, for service extraction only',
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource']) -> None:
@@ -79,7 +81,7 @@ def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource'])
 class GoogleAdwordsConnector(ToucanConnector):
     data_source_model: GoogleAdwordsDataSource
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     developer_token: str = None
     client_customer_id: str = None
     _oauth_trigger = 'instance'

toucan_connectors/google_analytics/google_analytics_connector.py

@@ -3,7 +3,7 @@
 import pandas as pd
 from apiclient.discovery import build
 from oauth2client.service_account import ServiceAccountCredentials
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from toucan_connectors.common import nosql_apply_parameters_to_query
 from toucan_connectors.google_credentials import GoogleCredentials
@@ -24,10 +24,7 @@ class DimensionFilter(BaseModel):
     operator: str
     expressions: List[str] = None
     caseSensitive: bool = False
-
-    class Config:
-        # TODO `not` param is not implemented
-        extra = 'allow'
+    model_config = ConfigDict(extra='allow')
 
 
 class DimensionFilterClause(BaseModel):
@@ -43,20 +40,14 @@ class DateRange(BaseModel):
 class Metric(BaseModel):
     expression: str
     alias: str = None
-
-    class Config:
-        # TODO `metricType` param is not implemented
-        extra = 'allow'
+    model_config = ConfigDict(extra='allow')
 
 
 class MetricFilter(BaseModel):
     metricName: str
     operator: str
     comparisonValue: str
-
-    class Config:
-        # TODO `not` param is not implemented
-        extra = 'allow'
+    model_config = ConfigDict(extra='allow')
 
 
 class MetricFilterClause(BaseModel):

toucan_connectors/google_big_query/google_big_query_connector.py

@@ -11,7 +11,7 @@
 from google.cloud.bigquery.dbapi import _helpers as bigquery_helpers
 from google.cloud.bigquery.job import QueryJob
 from google.oauth2.service_account import Credentials
-from pydantic import Field, create_model
+from pydantic import ConfigDict, Field, create_model
 
 from toucan_connectors.common import sanitize_query
 from toucan_connectors.google_credentials import GoogleCredentials, get_google_oauth2_credentials
@@ -103,10 +103,9 @@ class GoogleBigQueryConnector(ToucanConnector, DiscoverableConnector):
         'the Google APIs. For more information, see this '
         '<a href="https://developers.google.com/identity/protocols/googlescopes" target="_blank" >documentation</a>',
     )
-
-    class Config:
-        underscore_attrs_are_private = True
-        keep_untouched = (cached_property,)
+    # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property,))
 
     @staticmethod
     def _get_google_credentials(credentials: GoogleCredentials, scopes: List[str]) -> Credentials:

toucan_connectors/google_credentials.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field, HttpUrl, validator
+from pydantic import BaseModel, Field, HttpUrl, field_validator
 
 CREDENTIALS_INFO_MESSAGE = (
     'This information is provided in your '
@@ -42,7 +42,8 @@ class GoogleCredentials(BaseModel):
         description=CREDENTIALS_INFO_MESSAGE,
     )
 
-    @validator('private_key')
+    @field_validator('private_key')
+    @classmethod
     def unescape_break_lines(cls, v):
         """
         `private_key` is a long string like

toucan_connectors/google_sheets/google_sheets_connector.py

@@ -34,6 +34,8 @@ class GoogleSheetsDataSource(ToucanDataSource):
         True, title='Dates as floats', description='Render Date as Floats or String from the sheet'
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheetsDataSource']) -> None:

toucan_connectors/google_sheets_2/google_sheets_2_connector.py

@@ -67,6 +67,8 @@ class GoogleSheets2DataSource(ToucanDataSource):
     parameters: dict = Field(None, description='Additional URL parameters')
     parse_dates: List[str] = Field([], title='Dates column', description='Columns to parse as date')
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheets2DataSource']) -> None:
@@ -98,7 +100,7 @@ class GoogleSheets2Connector(ToucanConnector):
     _auth_flow = 'oauth2'
     _oauth_trigger = 'instance'
     oauth2_version = Field('1', **{'ui.hidden': True})
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
 
     # TODO: turn into a class property
     _baseroute = 'https://sheets.googleapis.com/v4/spreadsheets/'

toucan_connectors/http_api/http_api_connector.py

@@ -95,6 +95,8 @@ class HttpAPIDataSource(ToucanDataSource):
     filter: str = FilterSchema
     flatten_column: str = Field(None, description='Column containing nested rows')
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['HttpAPIDataSource']) -> None:

toucan_connectors/hubspot/hubspot_connector.py

@@ -70,7 +70,7 @@ class HubspotDataSource(ToucanDataSource):
 
 class HubspotConnector(ToucanConnector):
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     _oauth_trigger = 'instance'
     oauth2_version = Field('1', **{'ui.hidden': True})
     data_source_model: HubspotDataSource

toucan_connectors/hubspot_private_app/hubspot_connector.py

@@ -4,7 +4,7 @@
 
 import pandas as pd
 from hubspot import HubSpot  # type:ignore[import]
-from pydantic import BaseModel, Field, SecretStr
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
 
 from toucan_connectors.pagination import build_pagination_info
 from toucan_connectors.toucan_connector import DataSlice, ToucanConnector, ToucanDataSource
@@ -32,15 +32,11 @@ class _HubSpotPaging(BaseModel):
 
 
 class _HubSpotResult(BaseModel):
-    created_at: datetime | None
-    updated_at: datetime | None
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
     id_: str = Field(..., alias='id')
     properties: dict[str, Any] = Field(default_factory=dict)
-
-    # For basic_api objects, properties are in a 'properties' object. For specialized APIs, such as
-    # owners, they're keys of the root object
-    class Config:
-        extra = 'allow'
+    model_config = ConfigDict(extra='allow')
 
     def to_dict(self) -> dict[str, Any]:
         dict_ = self.dict(by_alias=True)

toucan_connectors/linkedinads/linkedinads_connector.py

@@ -73,6 +73,8 @@ class LinkedinadsDataSource(ToucanDataSource):
         description='See https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting for more information',
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['LinkedinadsDataSource']) -> None:
@@ -96,7 +98,7 @@ class LinkedinadsConnector(ToucanConnector):
     _auth_flow = 'oauth2'
     auth_flow_id: Optional[
         str
-    ]  # This ID is generated & provided to the data provider during the oauth authentication process
+    ] = None  # This ID is generated & provided to the data provider during the oauth authentication process
     _baseroute = 'https://api.linkedin.com/v2/adAnalyticsV2?q='
     template: Template = Field(
         None,

toucan_connectors/mongo/mongo_connector.py

@@ -5,7 +5,7 @@
 import pymongo
 from bson.son import SON
 from cached_property import cached_property
-from pydantic import Field, SecretStr, create_model, validator
+from pydantic import ConfigDict, Field, SecretStr, create_model, validator
 
 from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query
 from toucan_connectors.json_wrapper import JsonWrapper
@@ -152,10 +152,10 @@ class MongoConnector(ToucanConnector, VersionableEngineConnector):
     username: Optional[str] = Field(None, description='Your login username')
     password: Optional[SecretStr] = Field(None, description='Your login password')
     ssl: Optional[bool] = Field(None, description='Create the connection to the server using SSL')
+    model_config = ConfigDict(ignored_types=(cached_property, _lru_cache_wrapper))
 
-    class Config:
-        keep_untouched = (cached_property, _lru_cache_wrapper)
-
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator('password')
     def password_must_have_a_user(cls, password, values):
         if password is not None and values['username'] is None:

toucan_connectors/mysql/mysql_connector.py

@@ -7,7 +7,7 @@
 import pandas as pd
 import pymysql
 from cached_property import cached_property_with_ttl
-from pydantic import Field, SecretStr, constr, create_model, validator
+from pydantic import ConfigDict, Field, SecretStr, constr, create_model, validator
 from pymysql.constants import CR, ER
 
 from toucan_connectors.common import ConnectorStatus, pandas_read_sql
@@ -142,11 +142,14 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo
         description='SSL Mode to use to connect to the MySQL server. '
         'Equivalent of the --ssl-mode option of the MySQL client. Must be set in order to use SSL',
     )
+    # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = ConfigDict(
+        underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)
+    )
 
-    class Config:
-        underscore_attrs_are_private = True
-        keep_untouched = (cached_property_with_ttl,)
-
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator('ssl_key')
     @classmethod
     def ssl_key_validator(cls, ssl_key: str, values: dict) -> str:

toucan_connectors/one_drive/one_drive_connector.py

@@ -45,7 +45,7 @@ class OneDriveDataSource(ToucanDataSource):
         description='Read one sheet or append multiple sheets',
         placeholder='Enter a sheet or a comma separated list of sheets',
     )
-    range: Optional[str]
+    range: Optional[str] = None
     table: Optional[str] = Field(
         None,
         Title='Tables',
@@ -77,7 +77,7 @@ class OneDriveConnector(ToucanConnector):
     _auth_flow = 'oauth2'
     _oauth_trigger = 'connector'
     oauth2_version = Field('1', **{'ui.hidden': True})
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
 
     authorization_url: str = Field(None, **{'ui.hidden': True})
     token_url: str = Field(None, **{'ui.hidden': True})

toucan_connectors/pagination.py

@@ -10,7 +10,7 @@ class OffsetLimitInfo(BaseModel):
     """
 
     offset: int
-    limit: int | None
+    limit: int | None = None
 
 
 class UnknownSizeDatasetPaginationInfo(BaseModel):
@@ -48,8 +48,8 @@ class PaginationInfo(BaseModel):
 
     parameters: OffsetLimitInfo
     pagination_info: UnknownSizeDatasetPaginationInfo | KnownSizeDatasetPaginationInfo
-    next_page: OffsetLimitInfo | None
-    previous_page: OffsetLimitInfo | None
+    next_page: OffsetLimitInfo | None = None
+    previous_page: OffsetLimitInfo | None = None
 
 
 def build_pagination_info(

toucan_connectors/peakina/peakina_connector.py

@@ -2,13 +2,13 @@
 
 import pandas as pd
 from peakina.datasource import DataSource
+from pydantic import ConfigDict
 
 from toucan_connectors.toucan_connector import ToucanConnector
 
 
 class PeakinaDataSource(DataSource):
-    class Config:
-        extra = 'allow'
+    model_config = ConfigDict(extra='allow')
 
     def __init__(self, **data: Any) -> None:
         super().__init__(**data)