feat: migration code for pydanticv2 support + lint

toucan_connectors/adobe_analytics/adobe_analytics_connector.py

@@ -3,7 +3,6 @@
 
 import pandas as pd
 from adobe_analytics import Client, ReportDefinition
-from pydantic import ConfigDict, __version__ as pydantic_version
 
 from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource
 
@@ -43,8 +42,6 @@ def report_definition(self):
             source=self.source,
         )
 
-_PYDANTIC_VERSION_ONE = pydantic_version.startswith("1.")
-
 
 class AdobeAnalyticsConnector(ToucanConnector):
     """
@@ -63,6 +60,3 @@ def _retrieve_data(self, data_source: AdobeAnalyticsDataSource) -> pd.DataFrame:
         df = suites[data_source.suite_id].download(data_source.report_definition)
         df['suite_id'] = data_source.suite_id
         return df
-
-    if not _PYDANTIC_VERSION_ONE:
-        model_config = ConfigDict(arbitrary_types_allowed=True)

toucan_connectors/anaplan/anaplan_connector.py

@@ -90,7 +90,7 @@ class AnaplanAuthError(AnaplanError):
 class AnaplanConnector(ToucanConnector):
     data_source_model: AnaplanDataSource
     username: str
-    password: Optional[SecretStr]
+    password: Optional[SecretStr] = None
 
     def _extract_json(self, resp: requests.Response) -> dict:
         if resp.status_code in (401, 403):

toucan_connectors/awsathena/awsathena_connector.py

@@ -4,7 +4,7 @@
 import boto3
 import pandas as pd
 from cached_property import cached_property_with_ttl
-from pydantic import Field, SecretStr, constr, create_model
+from pydantic import ConfigDict, Field, SecretStr, constr, create_model
 
 from toucan_connectors.common import ConnectorStatus, apply_query_parameters, sanitize_query
 from toucan_connectors.pagination import build_pagination_info
@@ -77,10 +77,11 @@ class AwsathenaConnector(ToucanConnector, DiscoverableConnector):
     aws_access_key_id: str = Field(..., description='Your AWS access key ID')
     aws_secret_access_key: SecretStr = Field(None, description='Your AWS secret key')
     region_name: str = Field(..., description='Your AWS region name')
-
-    class Config:
-        underscore_attrs_are_private = True
-        keep_untouched = (cached_property_with_ttl,)
+    # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+    model_config = ConfigDict(
+        underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)
+    )
 
     def get_session(self) -> boto3.Session:
         return boto3.Session(

toucan_connectors/clickhouse/clickhouse_connector.py

@@ -24,6 +24,8 @@ class ClickhouseDataSource(ToucanDataSource):
         'your_table")',
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['ClickhouseDataSource']) -> None:

toucan_connectors/facebook_ads/facebook_ads_connector.py

@@ -78,7 +78,7 @@ def determine_query_params(self) -> Dict[str, str]:
 
 class FacebookAdsConnector(ToucanConnector):
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     _oauth_trigger = 'instance'
     data_source_model: FacebookAdsDataSource
     oauth2_version = Field('1', **{'ui.hidden': True})

toucan_connectors/github/github_connector.py

@@ -97,7 +97,7 @@ def get_form(cls, connector: 'GithubConnector', current_config, **kwargs):
 
 class GithubConnector(ToucanConnector):
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     data_source_model: GithubDataSource
     _oauth_trigger = 'instance'
     _oauth2_connector: OAuth2Connector = PrivateAttr()

toucan_connectors/google_adwords/google_adwords_connector.py

@@ -59,6 +59,8 @@ class GoogleAdwordsDataSource(ToucanDataSource):
         description='Max number of rows to extract, for service extraction only',
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource']) -> None:
@@ -79,7 +81,7 @@ def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource'])
 class GoogleAdwordsConnector(ToucanConnector):
     data_source_model: GoogleAdwordsDataSource
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     developer_token: str = None
     client_customer_id: str = None
     _oauth_trigger = 'instance'

toucan_connectors/google_analytics/google_analytics_connector.py

@@ -3,11 +3,15 @@
 import pandas as pd
 from apiclient.discovery import build
 from oauth2client.service_account import ServiceAccountCredentials
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from toucan_connectors.common import nosql_apply_parameters_to_query
 from toucan_connectors.google_credentials import GoogleCredentials
-from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource
+from toucan_connectors.toucan_connector import (
+    PYDANTIC_VERSION_ONE,
+    ToucanConnector,
+    ToucanDataSource,
+)
 
 API = 'analyticsreporting'
 SCOPE = 'https://www.googleapis.com/auth/analytics.readonly'
@@ -25,9 +29,15 @@ class DimensionFilter(BaseModel):
     expressions: List[str] = None
     caseSensitive: bool = False
 
-    class Config:
-        # TODO `not` param is not implemented
-        extra = 'allow'
+    # TODO[pydantic]: This is temporary, in the future we will only support V2
+    # and get rid of this condition + update the CI (link/test)
+    if PYDANTIC_VERSION_ONE:
+
+        class Config:
+            extra = 'allow'
+
+    else:
+        model_config = ConfigDict(extra='allow')
 
 
 class DimensionFilterClause(BaseModel):
@@ -44,19 +54,31 @@ class Metric(BaseModel):
     expression: str
     alias: str = None
 
-    class Config:
-        # TODO `metricType` param is not implemented
-        extra = 'allow'
+    # TODO[pydantic]: This is temporary, in the future we will only support V2
+    # and get rid of this condition + update the CI (link/test)
+    if PYDANTIC_VERSION_ONE:
+
+        class Config:
+            extra = 'allow'
+
+    else:
+        model_config = ConfigDict(extra='allow')
 
 
 class MetricFilter(BaseModel):
     metricName: str
     operator: str
     comparisonValue: str
 
-    class Config:
-        # TODO `not` param is not implemented
-        extra = 'allow'
+    # TODO[pydantic]: This is temporary, in the future we will only support V2
+    # and get rid of this condition + update the CI (link/test)
+    if PYDANTIC_VERSION_ONE:
+
+        class Config:
+            extra = 'allow'
+
+    else:
+        model_config = ConfigDict(extra='allow')
 
 
 class MetricFilterClause(BaseModel):

toucan_connectors/google_big_query/google_big_query_connector.py

@@ -11,11 +11,12 @@
 from google.cloud.bigquery.dbapi import _helpers as bigquery_helpers
 from google.cloud.bigquery.job import QueryJob
 from google.oauth2.service_account import Credentials
-from pydantic import Field, create_model
+from pydantic import ConfigDict, Field, create_model
 
 from toucan_connectors.common import sanitize_query
 from toucan_connectors.google_credentials import GoogleCredentials, get_google_oauth2_credentials
 from toucan_connectors.toucan_connector import (
+    PYDANTIC_VERSION_ONE,
     DiscoverableConnector,
     TableInfo,
     ToucanConnector,
@@ -104,9 +105,20 @@ class GoogleBigQueryConnector(ToucanConnector, DiscoverableConnector):
         '<a href="https://developers.google.com/identity/protocols/googlescopes" target="_blank" >documentation</a>',
     )
 
-    class Config:
-        underscore_attrs_are_private = True
-        keep_untouched = (cached_property,)
+    # TODO[pydantic]: This is temporary, in the future we will only support V2
+    # and get rid of this condition + update the CI (link/test)
+    if PYDANTIC_VERSION_ONE:
+
+        class Config:
+            underscore_attrs_are_private = True
+            ignored_types = (cached_property,)
+
+    else:
+        # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`.
+        # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
+        model_config = ConfigDict(
+            underscore_attrs_are_private=True, ignored_types=(cached_property,)
+        )
 
     @staticmethod
     def _get_google_credentials(credentials: GoogleCredentials, scopes: List[str]) -> Credentials:

toucan_connectors/google_credentials.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field, HttpUrl, validator
+from pydantic import BaseModel, Field, HttpUrl, field_validator
 
 CREDENTIALS_INFO_MESSAGE = (
     'This information is provided in your '
@@ -42,7 +42,8 @@ class GoogleCredentials(BaseModel):
         description=CREDENTIALS_INFO_MESSAGE,
     )
 
-    @validator('private_key')
+    @field_validator('private_key')
+    @classmethod
     def unescape_break_lines(cls, v):
         """
         `private_key` is a long string like

toucan_connectors/google_sheets/google_sheets_connector.py

@@ -34,6 +34,8 @@ class GoogleSheetsDataSource(ToucanDataSource):
         True, title='Dates as floats', description='Render Date as Floats or String from the sheet'
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheetsDataSource']) -> None:

toucan_connectors/google_sheets_2/google_sheets_2_connector.py

@@ -67,6 +67,8 @@ class GoogleSheets2DataSource(ToucanDataSource):
     parameters: dict = Field(None, description='Additional URL parameters')
     parse_dates: List[str] = Field([], title='Dates column', description='Columns to parse as date')
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheets2DataSource']) -> None:
@@ -98,7 +100,7 @@ class GoogleSheets2Connector(ToucanConnector):
     _auth_flow = 'oauth2'
     _oauth_trigger = 'instance'
     oauth2_version = Field('1', **{'ui.hidden': True})
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
 
     # TODO: turn into a class property
     _baseroute = 'https://sheets.googleapis.com/v4/spreadsheets/'

toucan_connectors/http_api/http_api_connector.py

@@ -95,6 +95,8 @@ class HttpAPIDataSource(ToucanDataSource):
     filter: str = FilterSchema
     flatten_column: str = Field(None, description='Column containing nested rows')
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['HttpAPIDataSource']) -> None:

toucan_connectors/hubspot/hubspot_connector.py

@@ -36,7 +36,7 @@
     'deals': {
         'url': 'https://api.hubapi.com/deals/v1/deal/paged',
         # 'url': 'https://api.hubapi.com/crm/v3/objects/deals',
-        'legacy': False,
+        'legacy': False,  # type: ignore [typeddict-item]
     },
     'products': {
         'url': 'https://api.hubapi.com/crm-objects/v1/objects/products/paged',
@@ -70,7 +70,7 @@ class HubspotDataSource(ToucanDataSource):
 
 class HubspotConnector(ToucanConnector):
     _auth_flow = 'oauth2'
-    auth_flow_id: Optional[str]
+    auth_flow_id: Optional[str] = None
     _oauth_trigger = 'instance'
     oauth2_version = Field('1', **{'ui.hidden': True})
     data_source_model: HubspotDataSource

toucan_connectors/hubspot_private_app/hubspot_connector.py

@@ -4,10 +4,15 @@
 
 import pandas as pd
 from hubspot import HubSpot  # type:ignore[import]
-from pydantic import BaseModel, Field, SecretStr
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
 
 from toucan_connectors.pagination import build_pagination_info
-from toucan_connectors.toucan_connector import DataSlice, ToucanConnector, ToucanDataSource
+from toucan_connectors.toucan_connector import (
+    PYDANTIC_VERSION_ONE,
+    DataSlice,
+    ToucanConnector,
+    ToucanDataSource,
+)
 
 
 class HubspotDataset(str, Enum):
@@ -32,15 +37,20 @@ class _HubSpotPaging(BaseModel):
 
 
 class _HubSpotResult(BaseModel):
-    created_at: datetime | None
-    updated_at: datetime | None
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
     id_: str = Field(..., alias='id')
     properties: dict[str, Any] = Field(default_factory=dict)
 
-    # For basic_api objects, properties are in a 'properties' object. For specialized APIs, such as
-    # owners, they're keys of the root object
-    class Config:
-        extra = 'allow'
+    # TODO[pydantic]: This is temporary, in the future we will only support V2
+    # and get rid of this condition + update the CI (link/test)
+    if PYDANTIC_VERSION_ONE:
+
+        class Config:
+            extra = 'allow'
+
+    else:
+        model_config = ConfigDict(extra='allow')
 
     def to_dict(self) -> dict[str, Any]:
         dict_ = self.dict(by_alias=True)

toucan_connectors/linkedinads/linkedinads_connector.py

@@ -73,6 +73,8 @@ class LinkedinadsDataSource(ToucanDataSource):
         description='See https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting for more information',
     )
 
+    # TODO[pydantic]: We need to refactor this class, by creating the `model_config` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     class Config:
         @staticmethod
         def schema_extra(schema: Dict[str, Any], model: Type['LinkedinadsDataSource']) -> None:
@@ -96,7 +98,7 @@ class LinkedinadsConnector(ToucanConnector):
     _auth_flow = 'oauth2'
     auth_flow_id: Optional[
         str
-    ]  # This ID is generated & provided to the data provider during the oauth authentication process
+    ] = None  # This ID is generated & provided to the data provider during the oauth authentication process
     _baseroute = 'https://api.linkedin.com/v2/adAnalyticsV2?q='
     template: Template = Field(
         None,

toucan_connectors/mongo/mongo_connector.py

@@ -5,13 +5,14 @@
 import pymongo
 from bson.son import SON
 from cached_property import cached_property
-from pydantic import Field, SecretStr, create_model, validator
+from pydantic import ConfigDict, Field, SecretStr, create_model, validator
 
 from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query
 from toucan_connectors.json_wrapper import JsonWrapper
 from toucan_connectors.mongo.mongo_translator import MongoConditionTranslator
 from toucan_connectors.pagination import build_pagination_info
 from toucan_connectors.toucan_connector import (
+    PYDANTIC_VERSION_ONE,
     DataSlice,
     ToucanConnector,
     ToucanDataSource,
@@ -152,10 +153,18 @@ class MongoConnector(ToucanConnector, VersionableEngineConnector):
     username: Optional[str] = Field(None, description='Your login username')
     password: Optional[SecretStr] = Field(None, description='Your login password')
     ssl: Optional[bool] = Field(None, description='Create the connection to the server using SSL')
+    # TODO[pydantic]: This is temporary, in the future we will only support V2
+    # and get rid of this condition + update the CI (link/test)
+    if PYDANTIC_VERSION_ONE:
 
-    class Config:
-        keep_untouched = (cached_property, _lru_cache_wrapper)
+        class Config:
+            ignored_types = (cached_property, _lru_cache_wrapper)
 
+    else:
+        model_config = ConfigDict(ignored_types=(cached_property, _lru_cache_wrapper))
+
+    # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually.
+    # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information.
     @validator('password')
     def password_must_have_a_user(cls, password, values):
         if password is not None and values['username'] is None: