Improve type checking in sources (#230)

* Fix outdated lockfile

* Remove sources/__init__ (allows mypy to run on pipeline scripts)

* Install missing stubs and ignore missing imports in specific libs

check-package.sh

@@ -19,11 +19,11 @@ while IFS= read -r d; do
       error="yes"
     fi
   fi
-done < <(find . -mindepth 1 -not -path "./docs/website*" -type d -regex "^./[^.^_].*" '!' -exec test -e "{}/__init__.py" ';' -print)
+done < <(find . -mindepth 1 -not -path "./sources" -not -path "./docs/website*" -type d -regex "^./[^.^_].*" '!' -exec test -e "{}/__init__.py" ';' -print)
 
 if [ -z $error ]; then
   exit 0
 fi
 
 # error in package
-exit 1
+exit 1

mypy.ini

@@ -1,6 +1,6 @@
 [mypy]
 python_version=3.8
-ignore_missing_imports=true
+ignore_missing_imports=false
 strict_optional=false
 warn_redundant_casts=true
 # disallow_any_explicit=true
@@ -16,5 +16,20 @@ warn_unused_ignores=true
 ;disallow_untyped_defs=false
 ;warn_return_any=false
 
+[mypy-facebook_business.*]
+ignore_missing_imports=true
+
+[mypy-asana.*]
+ignore_missing_imports=true
+
+[mypy-apiclient.*]
+ignore_missing_imports=true
+
+[mypy-simple_salesforce.*]
+ignore_missing_imports=true
+
+[mypy-proto.*]
+ignore_missing_imports=true
+
 [mypy-tests.*]
 ; ignore_errors=true

pyproject.toml

@@ -24,50 +24,38 @@ flake8-builtins = "^2.1.0"
 types-requests = "^2.28.11.7"
 mimesis = "^7.0.0"
 requests-mock = "^1.10.0"
+pandas-stubs = "^2.0.0"
 
 [tool.black]
 include = '.*py$'
 
-
-
 [tool.poetry.group.sql_database.dependencies]
 sqlalchemy = ">=1.4"
 pymysql = "^1.0.3"
 
-
-
 [tool.poetry.group.google_sheets.dependencies]
 google-api-python-client = "^2.78.0"
 
-
-
 [tool.poetry.group.google_analytics.dependencies]
 google-analytics-data = "^0.16.2"
 google-api-python-client = "^2.86.0"
 google-auth-oauthlib = "^1.0.0"
 requests-oauthlib = "^1.3.1"
 
-
-
 [tool.poetry.group.stripe_analytics.dependencies]
 pandas = "^2.0.0"
 stripe = "^5.0.0"
-
-
+types-stripe = "^3.5.2.14"
 
 [tool.poetry.group.asana_dlt.dependencies]
 asana = "^3.2.1"
 
-
 [tool.poetry.group.facebook_ads.dependencies]
 facebook-business = "^16.0.2"
 
-
-
 [tool.poetry.group.salesforce.dependencies]
 simple-salesforce = "^1.12.4"
 
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
-

sources/jira_pipeline.py

@@ -12,7 +12,7 @@ def load(endpoints: Optional[List[str]] = None) -> None:
         endpoints: A list of Jira endpoints. If not provided, defaults to all resources.
     """
     if not endpoints:
-        endpoints = jira().resources.keys()
+        endpoints = list(jira().resources.keys())
 
     pipeline = dlt.pipeline(
         pipeline_name="jira_pipeline", destination="duckdb", dataset_name="jira"

sources/stripe_analytics_pipeline.py

@@ -12,7 +12,7 @@
 
 
 def load_data(
-    endpoints: Tuple[str] = ENDPOINTS + INCREMENTAL_ENDPOINTS,
+    endpoints: Tuple[str, ...] = ENDPOINTS + INCREMENTAL_ENDPOINTS,
     start_date: Optional[DateTime] = None,
     end_date: Optional[DateTime] = None,
 ) -> None:
@@ -38,7 +38,7 @@ def load_data(
 
 
 def load_incremental_endpoints(
-    endpoints: Tuple[str] = INCREMENTAL_ENDPOINTS,
+    endpoints: Tuple[str, ...] = INCREMENTAL_ENDPOINTS,
     initial_start_date: Optional[DateTime] = None,
     end_date: Optional[DateTime] = None,
 ) -> None: