diff --git a/check-package.sh b/check-package.sh index 4b409a661..e80969564 100755 --- a/check-package.sh +++ b/check-package.sh @@ -19,11 +19,11 @@ while IFS= read -r d; do error="yes" fi fi -done < <(find . -mindepth 1 -not -path "./docs/website*" -type d -regex "^./[^.^_].*" '!' -exec test -e "{}/__init__.py" ';' -print) +done < <(find . -mindepth 1 -not -path "./sources" -not -path "./docs/website*" -type d -regex "^./[^.^_].*" '!' -exec test -e "{}/__init__.py" ';' -print) if [ -z $error ]; then exit 0 fi # error in package -exit 1 \ No newline at end of file +exit 1 diff --git a/mypy.ini b/mypy.ini index 33f3f0824..518e80eb8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,6 @@ [mypy] python_version=3.8 -ignore_missing_imports=true +ignore_missing_imports=false strict_optional=false warn_redundant_casts=true # disallow_any_explicit=true @@ -16,5 +16,20 @@ warn_unused_ignores=true ;disallow_untyped_defs=false ;warn_return_any=false +[mypy-facebook_business.*] +ignore_missing_imports=true + +[mypy-asana.*] +ignore_missing_imports=true + +[mypy-apiclient.*] +ignore_missing_imports=true + +[mypy-simple_salesforce.*] +ignore_missing_imports=true + +[mypy-proto.*] +ignore_missing_imports=true + [mypy-tests.*] ; ignore_errors=true diff --git a/poetry.lock b/poetry.lock index b64fc5cf6..bf4ed69ba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2133,6 +2133,21 @@ sql-other = ["SQLAlchemy (>=1.4.16)"] test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.6.3)"] +[[package]] +name = "pandas-stubs" +version = "2.0.2.230605" +description = "Type annotations for pandas" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pandas_stubs-2.0.2.230605-py3-none-any.whl", hash = "sha256:39106b602f3cb6dc5f728b84e1b32bde6ecf41ee34ee714c66228009609fbada"}, + {file = "pandas_stubs-2.0.2.230605.tar.gz", hash = "sha256:624c7bb06d38145a44b61be459ccd19b038e0bf20364a025ecaab78fea65e858"}, +] + +[package.dependencies] +numpy = ">=1.24.3" +types-pytz = ">=2022.1.1" + [[package]] name = "pathspec" version = "0.11.1" @@ -3135,6 +3150,17 @@ files = [ {file = "tomlkit-0.11.8.tar.gz", hash = "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3"}, ] +[[package]] +name = "types-pytz" +version = "2023.3.0.0" +description = "Typing stubs for pytz" +optional = false +python-versions = "*" +files = [ + {file = "types-pytz-2023.3.0.0.tar.gz", hash = "sha256:ecdc70d543aaf3616a7e48631543a884f74205f284cefd6649ddf44c6a820aac"}, + {file = "types_pytz-2023.3.0.0-py3-none-any.whl", hash = "sha256:4fc2a7fbbc315f0b6630e0b899fd6c743705abe1094d007b0e612d10da15e0f3"}, +] + [[package]] name = "types-requests" version = "2.30.0.0" @@ -3160,6 +3186,17 @@ files = [ {file = "types_setuptools-67.7.0.3-py3-none-any.whl", hash = "sha256:ba59707ba74243aebb0f764d4011f780f0d2dbf60cc1c3efcc20b875d93f43e1"}, ] +[[package]] +name = "types-stripe" +version = "3.5.2.14" +description = "Typing stubs for stripe" +optional = false +python-versions = "*" +files = [ + {file = "types-stripe-3.5.2.14.tar.gz", hash = "sha256:bcc020aa5ba9acd796b9f2ac21f044c8e377ce2c0f570057f0f64c4b4637bbe7"}, + {file = "types_stripe-3.5.2.14-py3-none-any.whl", hash = "sha256:f5f1249f72a35ada1db95523edc7e8f7b543dc8434b2ff23eaa9ec2e251c2e59"}, +] + [[package]] name = "types-urllib3" version = "1.26.25.13" @@ -3450,4 +3487,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "5666d06b2ada20ecd72d7bb270cd693fb2a966144ee5702956439d18c8bebe5b" +content-hash = "80393e8c9f6e95457743c63dc9a2be8aa3950613995d3479704e96568c3c32fb" diff --git a/pyproject.toml b/pyproject.toml index 4d11ada09..b642def1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,50 +24,38 @@ flake8-builtins = "^2.1.0" types-requests = "^2.28.11.7" mimesis = "^7.0.0" requests-mock = "^1.10.0" +pandas-stubs = "^2.0.0" [tool.black] include = '.*py$' - - [tool.poetry.group.sql_database.dependencies] sqlalchemy = ">=1.4" pymysql = "^1.0.3" - - [tool.poetry.group.google_sheets.dependencies] google-api-python-client = "^2.78.0" - - [tool.poetry.group.google_analytics.dependencies] google-analytics-data = "^0.16.2" google-api-python-client = "^2.86.0" google-auth-oauthlib = "^1.0.0" requests-oauthlib = "^1.3.1" - - [tool.poetry.group.stripe_analytics.dependencies] pandas = "^2.0.0" stripe = "^5.0.0" - - +types-stripe = "^3.5.2.14" [tool.poetry.group.asana_dlt.dependencies] asana = "^3.2.1" - [tool.poetry.group.facebook_ads.dependencies] facebook-business = "^16.0.2" - - [tool.poetry.group.salesforce.dependencies] simple-salesforce = "^1.12.4" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" - diff --git a/sources/__init__.py b/sources/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/sources/jira_pipeline.py b/sources/jira_pipeline.py index e18745cc0..cdc560b70 100644 --- a/sources/jira_pipeline.py +++ b/sources/jira_pipeline.py @@ -12,7 +12,7 @@ def load(endpoints: Optional[List[str]] = None) -> None: endpoints: A list of Jira endpoints. If not provided, defaults to all resources. """ if not endpoints: - endpoints = jira().resources.keys() + endpoints = list(jira().resources.keys()) pipeline = dlt.pipeline( pipeline_name="jira_pipeline", destination="duckdb", dataset_name="jira" diff --git a/sources/stripe_analytics_pipeline.py b/sources/stripe_analytics_pipeline.py index ba02bfbb6..2613b18dd 100644 --- a/sources/stripe_analytics_pipeline.py +++ b/sources/stripe_analytics_pipeline.py @@ -12,7 +12,7 @@ def load_data( - endpoints: Tuple[str] = ENDPOINTS + INCREMENTAL_ENDPOINTS, + endpoints: Tuple[str, ...] = ENDPOINTS + INCREMENTAL_ENDPOINTS, start_date: Optional[DateTime] = None, end_date: Optional[DateTime] = None, ) -> None: @@ -38,7 +38,7 @@ def load_data( def load_incremental_endpoints( - endpoints: Tuple[str] = INCREMENTAL_ENDPOINTS, + endpoints: Tuple[str, ...] = INCREMENTAL_ENDPOINTS, initial_start_date: Optional[DateTime] = None, end_date: Optional[DateTime] = None, ) -> None: