diff --git a/.github/workflows/ci-server.yml b/.github/workflows/ci-server.yml index 2ed4fd2087..c2105b35a6 100644 --- a/.github/workflows/ci-server.yml +++ b/.github/workflows/ci-server.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ['3.10', '3.11'] + python-version: ['3.11'] defaults: run: diff --git a/Dockerfile b/Dockerfile index 0485268c63..91ad5d3379 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ COPY playground/dist /weaverbird/playground/dist RUN yarn build -FROM python:3.10 as server +FROM python:3.11 as server WORKDIR /weaverbird/server diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 7d41420981..15a7d815d6 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog (weaverbird python package) +## Unreleased + +## Breaking + +- Dropped support for Python 3.10 +- Bumped `pydantic` minimum version from 1.x to 2.4.2 + ## [0.39.0] - 2023-10-16 ### Added diff --git a/server/playground.py b/server/playground.py index c9c3ae53c4..0bc7b68089 100644 --- a/server/playground.py +++ b/server/playground.py @@ -62,8 +62,7 @@ from weaverbird.backends.pypika_translator.translate import ( translate_pipeline as pypika_translate_pipeline, ) -from weaverbird.pipeline.pipeline import Pipeline -from weaverbird.pipeline.references import PipelineWithRefs +from weaverbird.pipeline.pipeline import Pipeline, PipelineWithRefs from weaverbird.pipeline.steps import DomainStep from weaverbird.pipeline.steps.utils.combination import Reference @@ -179,7 +178,7 @@ async def execute_pipeline(pipeline: Pipeline, **kwargs) -> str: limit=output.pop("limit"), total_rows=output.pop("total"), retrieved_rows=len(output["data"]), - ).dict() + ).model_dump() return json.dumps( { @@ -206,7 +205,7 @@ async def handle_pandas_backend_request(): return ( jsonify( { - "step": e.step.dict(), + "step": e.step.model_dump(), "index": e.index, "message": e.message, } @@ -346,7 +345,7 @@ def execute_mongo_aggregation_query(collection, query, limit, offset): async def dummy_reference_resolver(r: Reference) -> list[dict]: - return [DomainStep(domain=r.uid).dict()] + return [DomainStep(domain=r.uid).model_dump()] @app.route("/mongo", methods=["GET", "POST"]) @@ -379,7 +378,7 @@ async def handle_mongo_backend_request(): return jsonify( { - "pagination_info": pagination_info.dict(), + "pagination_info": pagination_info.model_dump(), "data": results["data"], "types": results["types"], "query": mongo_query, # provided for inspection purposes @@ -406,7 +405,7 @@ async def handle_mongo_translated_backend_request(): return jsonify( { - "pagination_info": pagination_info.dict(), + "pagination_info": pagination_info.model_dump(), "data": results["data"], "types": results["types"], "query": req_params["query"], # provided for inspection purposes @@ -519,7 +518,7 @@ async def handle_snowflake_backend_request(): limit=limit, retrieved_rows=len(df_results), total_rows=total_count, - ).dict(), + ).model_dump(), "schema": build_table_schema(df_results, index=False), "data": json.loads(df_results.to_json(orient="records")), "query": query, # provided for inspection purposes @@ -637,7 +636,7 @@ async def handle_postgres_backend_request(): limit=limit, retrieved_rows=len(query_results_page), total_rows=query_total_count, - ).dict(), + ).model_dump(), "results": { "headers": query_results_columns, "data": query_results_page, @@ -699,7 +698,7 @@ async def handle_athena_post_request(): return { "pagination_info": build_pagination_info( offset=offset, limit=limit, retrieved_rows=len(result), total_rows=None - ).dict(), + ).model_dump(), "results": { "headers": result.columns.to_list(), "data": json.loads(result.to_json(orient="records")), @@ -767,7 +766,7 @@ async def hangle_bigquery_post_request(): return { "pagination_info": build_pagination_info( offset=offset, limit=limit, retrieved_rows=len(result), total_rows=None - ).dict(), + ).model_dump(), "results": { "headers": result.columns.to_list(), "data": json.loads(result.to_json(orient="records")), @@ -834,7 +833,7 @@ async def handle_mysql_post_request(): return { "pagination_info": build_pagination_info( offset=offset, limit=limit, retrieved_rows=len(result), total_rows=None - ).dict(), + ).model_dump(), "results": { "headers": result.columns.to_list(), "data": json.loads(result[offset : offset + limit].to_json(orient="records")), diff --git a/server/poetry.lock b/server/poetry.lock index aaeac33e3d..3e49c4d32f 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -98,7 +98,6 @@ files = [ [package.dependencies] aiosignal = ">=1.1.2" -async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" @@ -121,6 +120,17 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "annotated-types" +version = "0.5.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.7" +files = [ + {file = "annotated_types-0.5.0-py3-none-any.whl", hash = "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd"}, + {file = "annotated_types-0.5.0.tar.gz", hash = "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802"}, +] + [[package]] name = "asn1crypto" version = "1.5.1" @@ -132,17 +142,6 @@ files = [ {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, ] -[[package]] -name = "async-timeout" -version = "4.0.3" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.7" -files = [ - {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, - {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, -] - [[package]] name = "attrs" version = "23.1.0" @@ -262,8 +261,6 @@ mypy-extensions = ">=0.4.3" packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -644,9 +641,6 @@ files = [ {file = "coverage-7.3.0.tar.gz", hash = "sha256:49dbb19cdcafc130f597d9e04a29d0a032ceedf729e41b181f51cd170e6ee865"}, ] -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - [package.extras] toml = ["tomli"] @@ -782,20 +776,6 @@ six = "*" [package.extras] test = ["coverage", "ipython", "numpydoc", "pytest", "traitlets"] -[[package]] -name = "exceptiongroup" -version = "1.1.3" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, - {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, -] - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "execnet" version = "2.0.2" @@ -821,9 +801,6 @@ files = [ {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"}, ] -[package.dependencies] -typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""} - [package.extras] docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] @@ -995,14 +972,8 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" -grpcio = [ - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, -] -grpcio-status = [ - {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, -] +grpcio = {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +grpcio-status = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -1053,10 +1024,7 @@ google-api-core = {version = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev", extras = google-cloud-bigquery-storage = {version = ">=2.6.0,<3.0.0dev", optional = true, markers = "extra == \"bqstorage\""} google-cloud-core = ">=1.6.0,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" -grpcio = [ - {version = ">=1.49.1,<2.0dev", markers = "python_version >= \"3.11\""}, - {version = ">=1.47.0,<2.0dev", markers = "python_version < \"3.11\""}, -] +grpcio = {version = ">=1.49.1,<2.0dev", markers = "python_version >= \"3.11\""} packaging = ">=20.0.0" pandas = {version = ">=1.1.0", optional = true, markers = "extra == \"pandas\""} proto-plus = ">=1.15.0,<2.0.0dev" @@ -1088,10 +1056,7 @@ files = [ [package.dependencies] google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} -proto-plus = [ - {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, -] +proto-plus = {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""} protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [package.extras] @@ -1434,8 +1399,6 @@ files = [ h11 = "*" h2 = ">=3.1.0" priority = "*" -taskgroup = {version = "*", markers = "python_version < \"3.11\""} -tomli = {version = "*", markers = "python_version < \"3.11\""} wsproto = ">=0.14.0" [package.extras] @@ -1921,7 +1884,6 @@ files = [ [package.dependencies] mypy-extensions = ">=1.0.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typing-extensions = ">=4.1.0" [package.extras] @@ -2051,10 +2013,7 @@ files = [ ] [package.dependencies] -numpy = [ - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, -] +numpy = {version = ">=1.23.2", markers = "python_version >= \"3.11\""} python-dateutil = ">=2.8.1" pytz = ">=2020.1" @@ -2342,55 +2301,140 @@ files = [ [[package]] name = "pydantic" -version = "1.10.13" -description = "Data validation and settings management using python type hints" +version = "2.4.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-2.4.2-py3-none-any.whl", hash = "sha256:bc3ddf669d234f4220e6e1c4d96b061abe0998185a8d7855c0126782b7abc8c1"}, + {file = "pydantic-2.4.2.tar.gz", hash = "sha256:94f336138093a5d7f426aac732dcfe7ab4eb4da243c88f891d65deb4a2556ee7"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.10.1" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.10.1" +description = "" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, - {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, - {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, - {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, - {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, - {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, - {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, - {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, - {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, - {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, -] - -[package.dependencies] -typing-extensions = ">=4.2.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] + {file = "pydantic_core-2.10.1-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:d64728ee14e667ba27c66314b7d880b8eeb050e58ffc5fec3b7a109f8cddbd63"}, + {file = "pydantic_core-2.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48525933fea744a3e7464c19bfede85df4aba79ce90c60b94d8b6e1eddd67096"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef337945bbd76cce390d1b2496ccf9f90b1c1242a3a7bc242ca4a9fc5993427a"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1392e0638af203cee360495fd2cfdd6054711f2db5175b6e9c3c461b76f5175"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0675ba5d22de54d07bccde38997e780044dcfa9a71aac9fd7d4d7a1d2e3e65f7"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:128552af70a64660f21cb0eb4876cbdadf1a1f9d5de820fed6421fa8de07c893"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f6e6aed5818c264412ac0598b581a002a9f050cb2637a84979859e70197aa9e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ecaac27da855b8d73f92123e5f03612b04c5632fd0a476e469dfc47cd37d6b2e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3c01c2fb081fced3bbb3da78510693dc7121bb893a1f0f5f4b48013201f362e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:92f675fefa977625105708492850bcbc1182bfc3e997f8eecb866d1927c98ae6"}, + {file = "pydantic_core-2.10.1-cp310-none-win32.whl", hash = "sha256:420a692b547736a8d8703c39ea935ab5d8f0d2573f8f123b0a294e49a73f214b"}, + {file = "pydantic_core-2.10.1-cp310-none-win_amd64.whl", hash = "sha256:0880e239827b4b5b3e2ce05e6b766a7414e5f5aedc4523be6b68cfbc7f61c5d0"}, + {file = "pydantic_core-2.10.1-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:073d4a470b195d2b2245d0343569aac7e979d3a0dcce6c7d2af6d8a920ad0bea"}, + {file = "pydantic_core-2.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:600d04a7b342363058b9190d4e929a8e2e715c5682a70cc37d5ded1e0dd370b4"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39215d809470f4c8d1881758575b2abfb80174a9e8daf8f33b1d4379357e417c"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eeb3d3d6b399ffe55f9a04e09e635554012f1980696d6b0aca3e6cf42a17a03b"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a7902bf75779bc12ccfc508bfb7a4c47063f748ea3de87135d433a4cca7a2f"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3625578b6010c65964d177626fde80cf60d7f2e297d56b925cb5cdeda6e9925a"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:caa48fc31fc7243e50188197b5f0c4228956f97b954f76da157aae7f67269ae8"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:07ec6d7d929ae9c68f716195ce15e745b3e8fa122fc67698ac6498d802ed0fa4"}, + {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e6f31a17acede6a8cd1ae2d123ce04d8cca74056c9d456075f4f6f85de055607"}, + {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d8f1ebca515a03e5654f88411420fea6380fc841d1bea08effb28184e3d4899f"}, + {file = "pydantic_core-2.10.1-cp311-none-win32.whl", hash = "sha256:6db2eb9654a85ada248afa5a6db5ff1cf0f7b16043a6b070adc4a5be68c716d6"}, + {file = "pydantic_core-2.10.1-cp311-none-win_amd64.whl", hash = "sha256:4a5be350f922430997f240d25f8219f93b0c81e15f7b30b868b2fddfc2d05f27"}, + {file = "pydantic_core-2.10.1-cp311-none-win_arm64.whl", hash = "sha256:5fdb39f67c779b183b0c853cd6b45f7db84b84e0571b3ef1c89cdb1dfc367325"}, + {file = "pydantic_core-2.10.1-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1f22a9ab44de5f082216270552aa54259db20189e68fc12484873d926426921"}, + {file = "pydantic_core-2.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8572cadbf4cfa95fb4187775b5ade2eaa93511f07947b38f4cd67cf10783b118"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db9a28c063c7c00844ae42a80203eb6d2d6bbb97070cfa00194dff40e6f545ab"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2a35baa428181cb2270a15864ec6286822d3576f2ed0f4cd7f0c1708472aff"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05560ab976012bf40f25d5225a58bfa649bb897b87192a36c6fef1ab132540d7"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6495008733c7521a89422d7a68efa0a0122c99a5861f06020ef5b1f51f9ba7c"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ac492c686defc8e6133e3a2d9eaf5261b3df26b8ae97450c1647286750b901"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8282bab177a9a3081fd3d0a0175a07a1e2bfb7fcbbd949519ea0980f8a07144d"}, + {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:aafdb89fdeb5fe165043896817eccd6434aee124d5ee9b354f92cd574ba5e78f"}, + {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f6defd966ca3b187ec6c366604e9296f585021d922e666b99c47e78738b5666c"}, + {file = "pydantic_core-2.10.1-cp312-none-win32.whl", hash = "sha256:7c4d1894fe112b0864c1fa75dffa045720a194b227bed12f4be7f6045b25209f"}, + {file = "pydantic_core-2.10.1-cp312-none-win_amd64.whl", hash = "sha256:5994985da903d0b8a08e4935c46ed8daf5be1cf217489e673910951dc533d430"}, + {file = "pydantic_core-2.10.1-cp312-none-win_arm64.whl", hash = "sha256:0d8a8adef23d86d8eceed3e32e9cca8879c7481c183f84ed1a8edc7df073af94"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:9badf8d45171d92387410b04639d73811b785b5161ecadabf056ea14d62d4ede"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:ebedb45b9feb7258fac0a268a3f6bec0a2ea4d9558f3d6f813f02ff3a6dc6698"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfe1090245c078720d250d19cb05d67e21a9cd7c257698ef139bc41cf6c27b4f"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e357571bb0efd65fd55f18db0a2fb0ed89d0bb1d41d906b138f088933ae618bb"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3dcd587b69bbf54fc04ca157c2323b8911033e827fffaecf0cafa5a892a0904"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c120c9ce3b163b985a3b966bb701114beb1da4b0468b9b236fc754783d85aa3"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15d6bca84ffc966cc9976b09a18cf9543ed4d4ecbd97e7086f9ce9327ea48891"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cabb9710f09d5d2e9e2748c3e3e20d991a4c5f96ed8f1132518f54ab2967221"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:82f55187a5bebae7d81d35b1e9aaea5e169d44819789837cdd4720d768c55d15"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1d40f55222b233e98e3921df7811c27567f0e1a4411b93d4c5c0f4ce131bc42f"}, + {file = "pydantic_core-2.10.1-cp37-none-win32.whl", hash = "sha256:14e09ff0b8fe6e46b93d36a878f6e4a3a98ba5303c76bb8e716f4878a3bee92c"}, + {file = "pydantic_core-2.10.1-cp37-none-win_amd64.whl", hash = "sha256:1396e81b83516b9d5c9e26a924fa69164156c148c717131f54f586485ac3c15e"}, + {file = "pydantic_core-2.10.1-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6835451b57c1b467b95ffb03a38bb75b52fb4dc2762bb1d9dbed8de31ea7d0fc"}, + {file = "pydantic_core-2.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b00bc4619f60c853556b35f83731bd817f989cba3e97dc792bb8c97941b8053a"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa467fd300a6f046bdb248d40cd015b21b7576c168a6bb20aa22e595c8ffcdd"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d99277877daf2efe074eae6338453a4ed54a2d93fb4678ddfe1209a0c93a2468"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa7db7558607afeccb33c0e4bf1c9a9a835e26599e76af6fe2fcea45904083a6"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aad7bd686363d1ce4ee930ad39f14e1673248373f4a9d74d2b9554f06199fb58"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:443fed67d33aa85357464f297e3d26e570267d1af6fef1c21ca50921d2976302"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:042462d8d6ba707fd3ce9649e7bf268633a41018d6a998fb5fbacb7e928a183e"}, + {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ecdbde46235f3d560b18be0cb706c8e8ad1b965e5c13bbba7450c86064e96561"}, + {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ed550ed05540c03f0e69e6d74ad58d026de61b9eaebebbaaf8873e585cbb18de"}, + {file = "pydantic_core-2.10.1-cp38-none-win32.whl", hash = "sha256:8cdbbd92154db2fec4ec973d45c565e767ddc20aa6dbaf50142676484cbff8ee"}, + {file = "pydantic_core-2.10.1-cp38-none-win_amd64.whl", hash = "sha256:9f6f3e2598604956480f6c8aa24a3384dbf6509fe995d97f6ca6103bb8c2534e"}, + {file = "pydantic_core-2.10.1-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:655f8f4c8d6a5963c9a0687793da37b9b681d9ad06f29438a3b2326d4e6b7970"}, + {file = "pydantic_core-2.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e570ffeb2170e116a5b17e83f19911020ac79d19c96f320cbfa1fa96b470185b"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64322bfa13e44c6c30c518729ef08fda6026b96d5c0be724b3c4ae4da939f875"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:485a91abe3a07c3a8d1e082ba29254eea3e2bb13cbbd4351ea4e5a21912cc9b0"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7c2b8eb9fc872e68b46eeaf835e86bccc3a58ba57d0eedc109cbb14177be531"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5cb87bdc2e5f620693148b5f8f842d293cae46c5f15a1b1bf7ceeed324a740c"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25bd966103890ccfa028841a8f30cebcf5875eeac8c4bde4fe221364c92f0c9a"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f323306d0556351735b54acbf82904fe30a27b6a7147153cbe6e19aaaa2aa429"}, + {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0c27f38dc4fbf07b358b2bc90edf35e82d1703e22ff2efa4af4ad5de1b3833e7"}, + {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f1365e032a477c1430cfe0cf2856679529a2331426f8081172c4a74186f1d595"}, + {file = "pydantic_core-2.10.1-cp39-none-win32.whl", hash = "sha256:a1c311fd06ab3b10805abb72109f01a134019739bd3286b8ae1bc2fc4e50c07a"}, + {file = "pydantic_core-2.10.1-cp39-none-win_amd64.whl", hash = "sha256:ae8a8843b11dc0b03b57b52793e391f0122e740de3df1474814c700d2622950a"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d43002441932f9a9ea5d6f9efaa2e21458221a3a4b417a14027a1d530201ef1b"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fcb83175cc4936a5425dde3356f079ae03c0802bbdf8ff82c035f8a54b333521"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:962ed72424bf1f72334e2f1e61b68f16c0e596f024ca7ac5daf229f7c26e4208"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cf5bb4dd67f20f3bbc1209ef572a259027c49e5ff694fa56bed62959b41e1f9"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e544246b859f17373bed915182ab841b80849ed9cf23f1f07b73b7c58baee5fb"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c0877239307b7e69d025b73774e88e86ce82f6ba6adf98f41069d5b0b78bd1bf"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:53df009d1e1ba40f696f8995683e067e3967101d4bb4ea6f667931b7d4a01357"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a1254357f7e4c82e77c348dabf2d55f1d14d19d91ff025004775e70a6ef40ada"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:524ff0ca3baea164d6d93a32c58ac79eca9f6cf713586fdc0adb66a8cdeab96a"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f0ac9fb8608dbc6eaf17956bf623c9119b4db7dbb511650910a82e261e6600f"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:320f14bd4542a04ab23747ff2c8a778bde727158b606e2661349557f0770711e"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:63974d168b6233b4ed6a0046296803cb13c56637a7b8106564ab575926572a55"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:417243bf599ba1f1fef2bb8c543ceb918676954734e2dcb82bf162ae9d7bd514"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dda81e5ec82485155a19d9624cfcca9be88a405e2857354e5b089c2a982144b2"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:14cfbb00959259e15d684505263d5a21732b31248a5dd4941f73a3be233865b9"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:631cb7415225954fdcc2a024119101946793e5923f6c4d73a5914d27eb3d3a05"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:bec7dd208a4182e99c5b6c501ce0b1f49de2802448d4056091f8e630b28e9a52"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:149b8a07712f45b332faee1a2258d8ef1fb4a36f88c0c17cb687f205c5dc6e7d"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d966c47f9dd73c2d32a809d2be529112d509321c5310ebf54076812e6ecd884"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7eb037106f5c6b3b0b864ad226b0b7ab58157124161d48e4b30c4a43fef8bc4b"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:154ea7c52e32dce13065dbb20a4a6f0cc012b4f667ac90d648d36b12007fa9f7"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e562617a45b5a9da5be4abe72b971d4f00bf8555eb29bb91ec2ef2be348cd132"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f23b55eb5464468f9e0e9a9935ce3ed2a870608d5f534025cd5536bca25b1402"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:e9121b4009339b0f751955baf4543a0bfd6bc3f8188f8056b1a25a2d45099934"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:0523aeb76e03f753b58be33b26540880bac5aa54422e4462404c432230543f33"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e0e2959ef5d5b8dc9ef21e1a305a21a36e254e6a34432d00c72a92fdc5ecda5"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da01bec0a26befab4898ed83b362993c844b9a607a86add78604186297eb047e"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f2e9072d71c1f6cfc79a36d4484c82823c560e6f5599c43c1ca6b5cdbd54f881"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f36a3489d9e28fe4b67be9992a23029c3cec0babc3bd9afb39f49844a8c721c5"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f64f82cc3443149292b32387086d02a6c7fb39b8781563e0ca7b8d7d9cf72bd7"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b4a6db486ac8e99ae696e09efc8b2b9fea67b63c8f88ba7a1a16c24a057a0776"}, + {file = "pydantic_core-2.10.1.tar.gz", hash = "sha256:0f8682dbdd2f67f8e1edddcbffcc29f60a6182b4901c367fc8c1c40d30bb0a82"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyjwt" @@ -2604,11 +2648,9 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] @@ -3251,20 +3293,6 @@ files = [ [package.extras] widechars = ["wcwidth"] -[[package]] -name = "taskgroup" -version = "0.0.0a4" -description = "backport of asyncio.TaskGroup, asyncio.Runner and asyncio.timeout" -optional = true -python-versions = "*" -files = [ - {file = "taskgroup-0.0.0a4-py2.py3-none-any.whl", hash = "sha256:5c1bd0e4c06114e7a4128583ab75c987597d5378a33948a3b74c662b90f61277"}, - {file = "taskgroup-0.0.0a4.tar.gz", hash = "sha256:eb08902d221e27661950f2a0320ddf3f939f579279996f81fe30779bca3a159c"}, -] - -[package.dependencies] -exceptiongroup = "*" - [[package]] name = "tenacity" version = "8.2.3" @@ -3290,17 +3318,6 @@ files = [ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - [[package]] name = "tomlkit" version = "0.12.1" @@ -3328,13 +3345,13 @@ requests = ">=2.23,<3.0" [[package]] name = "toucan-connectors" -version = "4.9.6" +version = "5.0.0" description = "Toucan Toco Connectors" optional = false python-versions = ">=3.10,<3.12" files = [ - {file = "toucan_connectors-4.9.6-py3-none-any.whl", hash = "sha256:4d81d160f774fa988e3de27bae830b01172e1571f513e078bcee9bc7842cc1ea"}, - {file = "toucan_connectors-4.9.6.tar.gz", hash = "sha256:2de19eb0c255f4341fdcc3954607cc1ed348324573475ba909ff9123e2a62b0e"}, + {file = "toucan_connectors-5.0.0-py3-none-any.whl", hash = "sha256:5bcd50cface9167f80ef3809cdcc264185c0663b09d2a700383871d8a1564a83"}, + {file = "toucan_connectors-5.0.0.tar.gz", hash = "sha256:e504833c2859a41a48a65cb510801034ce572abe30b6858c8404414c8d8acbaa"}, ] [package.dependencies] @@ -3347,7 +3364,7 @@ Jinja2 = ">=3.0.3,<4.0.0" jq = ">=1.2.2,<2.0.0" lxml = {version = ">=4.6.5,<5.0.0", optional = true, markers = "extra == \"redshift\" or extra == \"soap\" or extra == \"all\""} pyarrow = {version = "*", optional = true, markers = "extra == \"snowflake\" or extra == \"all\""} -pydantic = ">=1.9.1,<2.0.0" +pydantic = ">=2.4.2,<3.0.0" PyJWT = {version = ">=1.5.3,<3", optional = true, markers = "extra == \"snowflake\" or extra == \"all\""} pymongo = {version = ">=3.12.0", optional = true, markers = "extra == \"mongo\" or extra == \"all\""} PyMySQL = {version = ">=1.0.2,<2.0.0", optional = true, markers = "extra == \"google-cloud-mysql\" or extra == \"mysql\" or extra == \"all\""} @@ -3619,5 +3636,5 @@ pypika = ["PyPika"] [metadata] lock-version = "2.0" -python-versions = ">=3.10, <3.12" -content-hash = "64d43712918f139e6987b70c05138b878bab3d83e0ba877ab3a41fb6bbf16b9b" +python-versions = ">=3.11, <3.12" +content-hash = "06e4f01ba9605761f4088ef165bf890385fd7d88fc0776a34ec5afff66bc53dc" diff --git a/server/pyproject.toml b/server/pyproject.toml index ee4533216d..ed8588312d 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -12,8 +12,8 @@ license = "BSD-3-Clause" readme = "README.md" [tool.poetry.dependencies] -python = ">=3.10, <3.12" -pydantic = "^1.9.1" +python = ">=3.11, <3.12" +pydantic = "^2.4.2" # Dependencies for extras ## Pandas @@ -27,14 +27,16 @@ Quart-CORS = {version = ">=0.5,<0.8", optional = true} hypercorn = {version = ">=0.13,<0.16", optional = true} pymongo = {version = ">=4.2.0", optional = true, extras = ["srv", "tls"]} psycopg = {optional = true, version = "^3.0.15"} -toucan-connectors = {version = "^4.5.1", optional = true, extras = ["google_big_query", "mongo", "Redshift", "snowflake", "awsathena", "mysql"]} +toucan-connectors = {version = "^5.0.0", optional = true, extras = ["google_big_query", "mongo", "Redshift", "snowflake", "awsathena", "mysql"]} +pytest-asyncio = "^0.21.0" +python-dateutil = "^2.8.2" [tool.poetry.group.dev.dependencies] pytest-cov = "^4.1.0" pytest-mock = "^3.11.1" pytest-asyncio = "^0.21.0" # required so we can use nosql_apply_parameters_to_query during tests -toucan-connectors = { version = "^4.5.1", extras = ["google_big_query", "mongo", "Redshift", "snowflake", "awsathena"] } +toucan-connectors = {version = "^5.0.0", extras = ["google_big_query", "mongo", "Redshift", "snowflake", "awsathena", "mysql"]} pytest-benchmark = "^4.0.0" snowflake-sqlalchemy = "^1.5.0" types-python-dateutil = "^2.8.19" @@ -96,8 +98,8 @@ ignore = [ # Exclude a variety of commonly ignored directories. exclude = [".git", ".direnv"] -# Assume Python 3.10. -target-version = "py310" +# Assume Python 3.11. +target-version = "py311" [tool.ruff.flake8-quotes] docstring-quotes = "double" diff --git a/server/src/weaverbird/backends/mongo_translator/steps/append.py b/server/src/weaverbird/backends/mongo_translator/steps/append.py index 3f7e08f468..b93e34f647 100644 --- a/server/src/weaverbird/backends/mongo_translator/steps/append.py +++ b/server/src/weaverbird/backends/mongo_translator/steps/append.py @@ -27,10 +27,14 @@ def translate_append(step: AppendStep) -> list[MongoStep]: "References must be resolved before translating the pipeline" ) # noqa: B904 - domain_step = DomainStep(**sub_pipeline[0]) - pipeline_without_domain.steps = [ - getattr(steps, f"{s['name'].capitalize()}Step")(**s) for s in sub_pipeline[1:] - ] + if isinstance(sub_pipeline[0], DomainStep): + domain_step = sub_pipeline[0] + pipeline_without_domain.steps = [s.copy(deep=True) for s in sub_pipeline[1:]] + else: + domain_step = DomainStep(**sub_pipeline[0]) + pipeline_without_domain.steps = [ + getattr(steps, f"{s['name'].capitalize()}Step")(**s) for s in sub_pipeline[1:] + ] lookups.append( { "$lookup": { diff --git a/server/src/weaverbird/backends/mongo_translator/steps/join.py b/server/src/weaverbird/backends/mongo_translator/steps/join.py index afce1a61b2..a1ee2f991e 100644 --- a/server/src/weaverbird/backends/mongo_translator/steps/join.py +++ b/server/src/weaverbird/backends/mongo_translator/steps/join.py @@ -24,10 +24,14 @@ def translate_join(step: JoinStep) -> list[MongoStep]: raise Exception( # noqa: B904 "References must be resolved before translating the pipeline" ) - right_domain = DomainStep(**right[0]) - right_without_domain.steps = [ - getattr(steps, f"{s['name'].capitalize()}Step")(**s) for s in right[1:] - ] + if isinstance(right[0], DomainStep): + right_domain = right[0] + right_without_domain.steps = [s.copy(deep=True) for s in right[1:]] + else: + right_domain = DomainStep(**right[0]) + right_without_domain.steps = [ + getattr(steps, f"{s['name'].capitalize()}Step")(**s) for s in right[1:] + ] mongo_let: dict[str, str] = {} mongo_expr_and: list[dict[str, list[str]]] = [] diff --git a/server/src/weaverbird/backends/pandas_executor/pipeline_executor.py b/server/src/weaverbird/backends/pandas_executor/pipeline_executor.py index 5266096a87..9c628cb8ed 100644 --- a/server/src/weaverbird/backends/pandas_executor/pipeline_executor.py +++ b/server/src/weaverbird/backends/pandas_executor/pipeline_executor.py @@ -48,7 +48,7 @@ def execute_pipeline( "type": "pandas", "index": index + 1, "name": step.name, - "details": step.dict(), + "details": step.model_dump(), "elapsed_time": stopwatch.interval * 1000, "sizes": { "memory_used": convert_size(df.memory_usage(deep=True).sum()), diff --git a/server/src/weaverbird/backends/pandas_executor/steps/totals.py b/server/src/weaverbird/backends/pandas_executor/steps/totals.py index 23cbd4dc34..8a3f251055 100644 --- a/server/src/weaverbird/backends/pandas_executor/steps/totals.py +++ b/server/src/weaverbird/backends/pandas_executor/steps/totals.py @@ -19,7 +19,7 @@ def get_total_for_dimension( ] aggregations = [] for aggregation in step.aggregations: - agg = aggregation.copy() + agg = aggregation.model_copy(deep=True) agg.columns = agg.new_columns aggregations.append(agg) diff --git a/server/src/weaverbird/backends/pandas_executor/steps/utils/condition.py b/server/src/weaverbird/backends/pandas_executor/steps/utils/condition.py index c6c72a2eec..fb2df1e035 100644 --- a/server/src/weaverbird/backends/pandas_executor/steps/utils/condition.py +++ b/server/src/weaverbird/backends/pandas_executor/steps/utils/condition.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import UTC, datetime from numpy.ma import logical_and, logical_or from pandas import DataFrame, Series, Timestamp @@ -23,12 +23,12 @@ def _date_bound_condition_to_tz_aware_timestamp(value: str | RelativeDate | date if isinstance(value, RelativeDate): value = evaluate_relative_date(value) if isinstance(value, datetime): - tz = value.tzinfo or "UTC" + tz = value.tzinfo or UTC # Cannot pass a tz-aware datetime object with tz arg return Timestamp(value.replace(tzinfo=None), tz=tz) else: # str ts = Timestamp(value) - return ts if ts.tzinfo else ts.replace(tz="UTC") + return ts if ts.tzinfo else ts.replace(tzinfo=UTC) def apply_condition(condition: Condition, df: DataFrame) -> Series: diff --git a/server/src/weaverbird/backends/pandas_executor/steps/utils/dates.py b/server/src/weaverbird/backends/pandas_executor/steps/utils/dates.py index c12e21374c..7125a0082a 100644 --- a/server/src/weaverbird/backends/pandas_executor/steps/utils/dates.py +++ b/server/src/weaverbird/backends/pandas_executor/steps/utils/dates.py @@ -24,7 +24,4 @@ def evaluate_relative_date(relative_date: RelativeDate) -> datetime: quantity = relative_date.quantity duration = relative_date.duration + "s" - return operation( - relative_date.date, - relativedelta(**{duration: quantity}), # type: ignore - ) + return operation(relative_date.date, relativedelta(**{duration: quantity})) diff --git a/server/src/weaverbird/backends/pandas_executor/types.py b/server/src/weaverbird/backends/pandas_executor/types.py index d475b0ed8c..f997efc407 100644 --- a/server/src/weaverbird/backends/pandas_executor/types.py +++ b/server/src/weaverbird/backends/pandas_executor/types.py @@ -15,7 +15,7 @@ class StepExecutionReport(BaseModel): class PipelineExecutionReport(BaseModel): - steps_reports: list[StepExecutionReport] = Field(min_items=0) + steps_reports: list[StepExecutionReport] = Field(min_length=0) DomainRetriever = Callable[[str | Reference], DataFrame] diff --git a/server/src/weaverbird/backends/pypika_translator/translators/base.py b/server/src/weaverbird/backends/pypika_translator/translators/base.py index f606449ded..a294c56dff 100644 --- a/server/src/weaverbird/backends/pypika_translator/translators/base.py +++ b/server/src/weaverbird/backends/pypika_translator/translators/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from functools import cache from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union, cast, get_args @@ -256,7 +256,7 @@ def _merge_first_steps( and isinstance(second_step, TopStep) and self._source_rows_subset < second_step.limit ): - second_step = second_step.copy(update={"limit": self._source_rows_subset}) + second_step = second_step.model_copy(update={"limit": self._source_rows_subset}) ctx = step_method(step=second_step, prev_step_table=table, builder=None, columns=columns) @@ -1156,11 +1156,7 @@ def _get_single_condition_criterion( dt = condition.value else: dt = dateutil_parser.parse(condition.value) - dt = ( - dt.replace(tzinfo=timezone.utc) - if dt.tzinfo is None - else dt.astimezone(timezone.utc) - ) + dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC) value_to_compare = self._cast_to_timestamp(dt.strftime("%Y-%m-%d %H:%M:%S")) elif isinstance(condition.value, functions.Function): diff --git a/server/src/weaverbird/pipeline/conditions.py b/server/src/weaverbird/pipeline/conditions.py index 209a9f172b..f422deb473 100644 --- a/server/src/weaverbird/pipeline/conditions.py +++ b/server/src/weaverbird/pipeline/conditions.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Annotated, Any, Literal -from pydantic import BaseConfig, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from weaverbird.pipeline.dates import RelativeDate from weaverbird.pipeline.types import ColumnName @@ -32,7 +32,7 @@ class NullCondition(BaseCondition): class MatchCondition(BaseCondition): column: ColumnName operator: Literal["matches", "notmatches"] - value: str + value: str | int | float class DateBoundCondition(BaseModel): @@ -48,11 +48,10 @@ class DateBoundCondition(BaseModel): class BaseConditionCombo(BaseCondition, ABC): - class Config(BaseConfig): - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) def to_dict(self): - return self.dict(by_alias=True) + return self.model_dump(by_alias=True) class ConditionComboAnd(BaseConditionCombo): @@ -64,5 +63,5 @@ class ConditionComboOr(BaseConditionCombo): Condition = ConditionComboAnd | ConditionComboOr | SimpleCondition -ConditionComboOr.update_forward_refs() -ConditionComboAnd.update_forward_refs() +ConditionComboOr.model_rebuild() +ConditionComboAnd.model_rebuild() diff --git a/server/src/weaverbird/pipeline/dates.py b/server/src/weaverbird/pipeline/dates.py index 13b3f34b86..8215a68d53 100644 --- a/server/src/weaverbird/pipeline/dates.py +++ b/server/src/weaverbird/pipeline/dates.py @@ -1,11 +1,22 @@ from datetime import datetime -from typing import Literal +from typing import Any, Literal -from pydantic import BaseModel +from dateutil.parser import parse as parse_dt +from pydantic import BaseModel, field_validator class RelativeDate(BaseModel): - date: datetime | str | None + date: datetime | None = None operator: Literal["from", "until", "before", "after"] quantity: int duration: Literal["year", "quarter", "month", "week", "day"] + + @field_validator("date", mode="before") + @classmethod + def _convert_date_to_datetime(cls, value: Any) -> Any: + if isinstance(value, str): + try: + return parse_dt(value) + except Exception as exc: + raise ValueError(f"Invalid value provided: {value}") from exc + return value diff --git a/server/src/weaverbird/pipeline/formula_ast/types.py b/server/src/weaverbird/pipeline/formula_ast/types.py index 7ef74ea4b5..4501f7a33f 100644 --- a/server/src/weaverbird/pipeline/formula_ast/types.py +++ b/server/src/weaverbird/pipeline/formula_ast/types.py @@ -26,13 +26,13 @@ class ColumnName(BaseModel): # Dataclasses do not supported recursive types for now -class Operation(BaseModel, smart_union=True): +class Operation(BaseModel): left: Expression right: Expression operator: Operator -Operation.update_forward_refs() +Operation.model_rebuild() def format_expr( diff --git a/server/src/weaverbird/pipeline/pipeline.py b/server/src/weaverbird/pipeline/pipeline.py index fc51d68af3..2f98c79101 100644 --- a/server/src/weaverbird/pipeline/pipeline.py +++ b/server/src/weaverbird/pipeline/pipeline.py @@ -12,7 +12,11 @@ InclusionCondition, MatchCondition, ) +from weaverbird.pipeline.steps.append import AppendStepWithRefs +from weaverbird.pipeline.steps.domain import DomainStepWithRef from weaverbird.pipeline.steps.hierarchy import HierarchyStep +from weaverbird.pipeline.steps.join import JoinStepWithRef +from weaverbird.pipeline.steps.utils.combination import ReferenceResolver from .steps import ( AbsoluteValueStep, @@ -155,8 +159,11 @@ class Pipeline(BaseModel): steps: list[PipelineStep] + def model_dump(self, *, exclude_none: bool = True, **kwargs) -> dict: + return super().model_dump(exclude_none=exclude_none, **kwargs) + def dict(self, *, exclude_none: bool = True, **kwargs) -> dict: - return super().dict(exclude_none=True, **kwargs) + return self.model_dump(exclude_none=exclude_none, **kwargs) PipelineStepWithVariables = Annotated[ @@ -395,4 +402,46 @@ def render(self, variables: dict[str, Any], renderer) -> Pipeline: return Pipeline(steps=steps_rendered) -PipelineWithVariables.update_forward_refs() +PipelineStepWithRefs = Annotated[ + AppendStepWithRefs | DomainStepWithRef | JoinStepWithRef, + Field(discriminator="name"), +] + + +class PipelineWithRefs(BaseModel): + """ + Represents a pipeline in which some steps can reference some other pipelines using the syntax + `{"type": "ref", "uid": "..."}` + """ + + steps: list[PipelineStepWithRefs | PipelineStep | PipelineStepWithVariables] + + async def resolve_references( + self, reference_resolver: ReferenceResolver + ) -> PipelineWithVariables | None: + """ + Walk the pipeline steps and replace any reference by its corresponding pipeline. + The sub-pipelines added should also be handled, so that they will be no references anymore in the result. + """ + resolved_steps: list[PipelineStepWithRefs | PipelineStepWithVariables | PipelineStep] = [] + for step in self.steps: + resolved_step = ( + await step.resolve_references(reference_resolver) + if hasattr(step, "resolve_references") + else step + ) + if isinstance(resolved_step, PipelineWithVariables): + resolved_steps.extend(resolved_step.steps) + elif resolved_step is not None: # None means the step should be skipped + resolved_steps.append(resolved_step) + + return PipelineWithVariables(steps=resolved_steps) + + +PipelineWithVariables.model_rebuild() + + +class ReferenceUnresolved(Exception): + """ + Raised when a mandatory reference is not resolved + """ diff --git a/server/src/weaverbird/pipeline/references.py b/server/src/weaverbird/pipeline/references.py deleted file mode 100644 index 45f3d71c69..0000000000 --- a/server/src/weaverbird/pipeline/references.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Annotated - -from pydantic import BaseModel, Field - -from .pipeline import PipelineStep, PipelineStepWithVariables, PipelineWithVariables -from .steps import AppendStepWithRefs, DomainStepWithRef, JoinStepWithRef -from .steps.utils.combination import ReferenceResolver - -PipelineStepWithRefs = Annotated[ - AppendStepWithRefs | DomainStepWithRef | JoinStepWithRef, - Field(discriminator="name"), # noqa: F821 -] - - -class PipelineWithRefs(BaseModel): - """ - Represents a pipeline in which some steps can reference some other pipelines using the syntax - `{"type": "ref", "uid": "..."}` - """ - - steps: list[PipelineStepWithRefs | PipelineStepWithVariables | PipelineStep] - - async def resolve_references( - self, reference_resolver: ReferenceResolver - ) -> PipelineWithVariables | None: - """ - Walk the pipeline steps and replace any reference by its corresponding pipeline. - The sub-pipelines added should also be handled, so that they will be no references anymore in the result. - """ - resolved_steps: list[PipelineStepWithRefs | PipelineStepWithVariables | PipelineStep] = [] - for step in self.steps: - resolved_step = ( - await step.resolve_references(reference_resolver) - if hasattr(step, "resolve_references") - else step - ) - if isinstance(resolved_step, PipelineWithVariables): - resolved_steps.extend(resolved_step.steps) - elif resolved_step is not None: # None means the step should be skipped - resolved_steps.append(resolved_step) - - return PipelineWithVariables(steps=resolved_steps) - - -class ReferenceUnresolved(Exception): - """ - Raised when a mandatory reference is not resolved - """ diff --git a/server/src/weaverbird/pipeline/steps/aggregate.py b/server/src/weaverbird/pipeline/steps/aggregate.py index 5087bcb580..3aba375977 100644 --- a/server/src/weaverbird/pipeline/steps/aggregate.py +++ b/server/src/weaverbird/pipeline/steps/aggregate.py @@ -1,7 +1,7 @@ from collections.abc import Sequence from typing import Literal -from pydantic import BaseConfig, BaseModel, Field, root_validator, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.steps.utils.render_variables import StepWithVariablesMixin @@ -25,15 +25,15 @@ class Aggregation(BaseModel): new_columns: list[ColumnName] = Field(alias="newcolumns") agg_function: AggregateFn = Field(alias="aggfunction") columns: list[ColumnName] + model_config = ConfigDict(populate_by_name=True) - class Config(BaseConfig): - allow_population_by_field_name = True - - @validator("columns", pre=True) + @field_validator("columns", mode="before") + @classmethod def validate_unique_columns(cls, value): return validate_unique_columns(value) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def handle_legacy_syntax(cls, values): if "column" in values: values["columns"] = [values.pop("column")] diff --git a/server/src/weaverbird/pipeline/steps/append.py b/server/src/weaverbird/pipeline/steps/append.py index a29520d32c..1e9f04fb40 100644 --- a/server/src/weaverbird/pipeline/steps/append.py +++ b/server/src/weaverbird/pipeline/steps/append.py @@ -5,7 +5,7 @@ from .utils.combination import ( PipelineOrDomainName, - PipelineOrDomainNameOrReference, + PipelineWithRefsOrDomainNameOrReference, ReferenceResolver, resolve_if_reference, ) @@ -24,7 +24,7 @@ class AppendStepWithVariable(AppendStep, StepWithVariablesMixin): class AppendStepWithRefs(BaseAppendStep): - pipelines: list[PipelineOrDomainNameOrReference] + pipelines: list[PipelineWithRefsOrDomainNameOrReference] async def resolve_references( self, reference_resolver: ReferenceResolver diff --git a/server/src/weaverbird/pipeline/steps/concatenate.py b/server/src/weaverbird/pipeline/steps/concatenate.py index 0235d54673..2ae0854b66 100644 --- a/server/src/weaverbird/pipeline/steps/concatenate.py +++ b/server/src/weaverbird/pipeline/steps/concatenate.py @@ -9,7 +9,7 @@ class ConcatenateStep(BaseStep): name: Literal["concatenate"] = "concatenate" - columns: list[ColumnName] = Field(..., min_items=2) + columns: list[ColumnName] = Field(..., min_length=2) separator: str new_column_name: ColumnName diff --git a/server/src/weaverbird/pipeline/steps/cumsum.py b/server/src/weaverbird/pipeline/steps/cumsum.py index d125866f8b..d85ff59fca 100644 --- a/server/src/weaverbird/pipeline/steps/cumsum.py +++ b/server/src/weaverbird/pipeline/steps/cumsum.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import Field, root_validator +from pydantic import Field, model_validator from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.steps.utils.render_variables import StepWithVariablesMixin @@ -15,9 +15,10 @@ class CumSumStep(BaseStep): alias="toCumSum", ) reference_column: ColumnName - groupby: list[ColumnName] | None + groupby: list[ColumnName] | None = None - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def handle_legacy_syntax(cls, values): if "valueColumn" in values: values["value_column"] = values.pop("valueColumn") diff --git a/server/src/weaverbird/pipeline/steps/customsql.py b/server/src/weaverbird/pipeline/steps/customsql.py index b18766c96d..a98949cf34 100644 --- a/server/src/weaverbird/pipeline/steps/customsql.py +++ b/server/src/weaverbird/pipeline/steps/customsql.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import validator +from pydantic import field_validator from weaverbird.pipeline.steps.utils.base import BaseStep @@ -14,7 +14,8 @@ class CustomSqlStep(BaseStep): def _strip_query(query: str) -> str: return query.strip().strip(";") - @validator("query") + @field_validator("query") + @classmethod def _validate_query(cls, query: str) -> str: assert ";" not in ( stripped := cls._strip_query(query) diff --git a/server/src/weaverbird/pipeline/steps/date_extract.py b/server/src/weaverbird/pipeline/steps/date_extract.py index 68262a3e6f..7a71382762 100644 --- a/server/src/weaverbird/pipeline/steps/date_extract.py +++ b/server/src/weaverbird/pipeline/steps/date_extract.py @@ -60,8 +60,8 @@ class DateExtractStep(BaseStep): column: str date_info: list[DATE_INFO] = Field(default_factory=list) new_columns: list[ColumnName] = Field(default_factory=list) - operation: BASIC_DATE_PARTS | None - new_column_name: ColumnName | None + operation: BASIC_DATE_PARTS | None = None + new_column_name: ColumnName | None = None class DateExtractStepWithVariable(DateExtractStep, StepWithVariablesMixin): diff --git a/server/src/weaverbird/pipeline/steps/dissolve.py b/server/src/weaverbird/pipeline/steps/dissolve.py index 17737215aa..d14cf42d51 100644 --- a/server/src/weaverbird/pipeline/steps/dissolve.py +++ b/server/src/weaverbird/pipeline/steps/dissolve.py @@ -1,7 +1,7 @@ from itertools import chain as ichain from typing import Any, Literal -from pydantic import validator +from pydantic import field_validator from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.types import ColumnName @@ -20,12 +20,14 @@ def _ensure_unique_and_non_empty(values: list[Any], col_name: str) -> None: assert all(len(v) for v in values), f"all values in '{col_name}' must be non-empty" assert len(values) == len(set(values)), f"all values in '{col_name}' must be unique" - @validator("groups") + @field_validator("groups") + @classmethod def _len_validator(cls, values: list) -> list: assert len(values) > 0, "list must contain at least one element" return values - @validator("aggregations") + @field_validator("aggregations") + @classmethod def _validate_aggregations(cls, values: list[Aggregation]) -> list[Aggregation]: if len(values) < 1: return values diff --git a/server/src/weaverbird/pipeline/steps/domain.py b/server/src/weaverbird/pipeline/steps/domain.py index 45a4272a45..625106ecc8 100644 --- a/server/src/weaverbird/pipeline/steps/domain.py +++ b/server/src/weaverbird/pipeline/steps/domain.py @@ -30,14 +30,12 @@ async def resolve_references( Not that the resulting array must be flattened: it should look like [step 1, step 2, step 3], not [[step 1, step 2], step 3] """ - from weaverbird.pipeline.references import PipelineWithRefs + from weaverbird.pipeline.pipeline import PipelineWithRefs, ReferenceUnresolved resolved = await resolve_if_reference(reference_resolver, self.domain) if isinstance(resolved, list): return await PipelineWithRefs(steps=resolved).resolve_references(reference_resolver) elif resolved is None: - from weaverbird.pipeline.references import ReferenceUnresolved - raise ReferenceUnresolved() else: return DomainStep( diff --git a/server/src/weaverbird/pipeline/steps/evolution.py b/server/src/weaverbird/pipeline/steps/evolution.py index 6eec5b207f..913628bbcb 100644 --- a/server/src/weaverbird/pipeline/steps/evolution.py +++ b/server/src/weaverbird/pipeline/steps/evolution.py @@ -17,7 +17,7 @@ class EvolutionStep(BaseStep): evolution_type: EVOLUTION_TYPE evolution_format: EVOLUTION_FORMAT index_columns: list[str] = Field(default_factory=list) - new_column: str | None + new_column: str | None = None class EvolutionStepWithVariable(EvolutionStep, StepWithVariablesMixin): diff --git a/server/src/weaverbird/pipeline/steps/fillna.py b/server/src/weaverbird/pipeline/steps/fillna.py index 2611292bb6..c9944e9238 100644 --- a/server/src/weaverbird/pipeline/steps/fillna.py +++ b/server/src/weaverbird/pipeline/steps/fillna.py @@ -1,6 +1,6 @@ from typing import Any, Literal -from pydantic import Field, root_validator +from pydantic import Field, model_validator from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.steps.utils.render_variables import StepWithVariablesMixin @@ -9,10 +9,11 @@ class FillnaStep(BaseStep): name: Literal["fillna"] = "fillna" - columns: list[ColumnName] = Field(min_items=1) + columns: list[ColumnName] = Field(min_length=1) value: Any - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def handle_legacy_syntax(cls, values): if "column" in values: values["columns"] = [values.pop("column")] diff --git a/server/src/weaverbird/pipeline/steps/hierarchy.py b/server/src/weaverbird/pipeline/steps/hierarchy.py index 0eca9404d4..35ae636b3b 100644 --- a/server/src/weaverbird/pipeline/steps/hierarchy.py +++ b/server/src/weaverbird/pipeline/steps/hierarchy.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import validator +from pydantic import field_validator from weaverbird.pipeline.steps.utils.base import BaseStep @@ -11,7 +11,8 @@ class HierarchyStep(BaseStep): hierarchy: list[str] include_nulls: bool = False - @validator("hierarchy") + @field_validator("hierarchy") + @classmethod def _ensure_hierarchy(cls, values: list[str]) -> list[str]: assert len(values) > 0, "at least one value must be specified" return values diff --git a/server/src/weaverbird/pipeline/steps/ifthenelse.py b/server/src/weaverbird/pipeline/steps/ifthenelse.py index 2c039084d3..b4032dd7df 100644 --- a/server/src/weaverbird/pipeline/steps/ifthenelse.py +++ b/server/src/weaverbird/pipeline/steps/ifthenelse.py @@ -19,7 +19,7 @@ class IfThenElse(BaseModel): name: Literal["ifthenelse"] = "ifthenelse" -IfThenElse.update_forward_refs() +IfThenElse.model_rebuild() class IfthenelseStep(BaseStep, IfThenElse): diff --git a/server/src/weaverbird/pipeline/steps/join.py b/server/src/weaverbird/pipeline/steps/join.py index 4e4b9e07d4..676cb5b63f 100644 --- a/server/src/weaverbird/pipeline/steps/join.py +++ b/server/src/weaverbird/pipeline/steps/join.py @@ -8,7 +8,7 @@ from .utils.combination import ( PipelineOrDomainName, - PipelineOrDomainNameOrReference, + PipelineWithRefsOrDomainNameOrReference, ReferenceResolver, resolve_if_reference, ) @@ -19,7 +19,7 @@ class BaseJoinStep(BaseStep): name: Literal["join"] = "join" type: Literal["left", "inner", "left outer"] - on: list[JoinColumnsPair] = Field(..., min_items=1) + on: list[JoinColumnsPair] = Field(..., min_length=1) class JoinStep(BaseJoinStep): @@ -31,14 +31,14 @@ class JoinStepWithVariable(JoinStep, StepWithVariablesMixin): class JoinStepWithRef(BaseJoinStep): - right_pipeline: PipelineOrDomainNameOrReference + right_pipeline: PipelineWithRefsOrDomainNameOrReference async def resolve_references( self, reference_resolver: ReferenceResolver ) -> JoinStepWithVariable | None: right_pipeline = await resolve_if_reference(reference_resolver, self.right_pipeline) if right_pipeline is None: - from ..references import ReferenceUnresolved + from weaverbird.pipeline.pipeline import ReferenceUnresolved raise ReferenceUnresolved() return JoinStepWithVariable( diff --git a/server/src/weaverbird/pipeline/steps/percentage.py b/server/src/weaverbird/pipeline/steps/percentage.py index a266ce0ca0..9d52c79124 100644 --- a/server/src/weaverbird/pipeline/steps/percentage.py +++ b/server/src/weaverbird/pipeline/steps/percentage.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import Field, validator +from pydantic import Field, model_validator from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.types import ColumnName @@ -12,8 +12,8 @@ class PercentageStep(BaseStep): group: list[ColumnName] = Field(default_factory=list) new_column_name: ColumnName | None = None - @validator("new_column_name", always=True) - def set_new_column_name_if_not_set(cls, v, values): - if v is None: - return f"{values['column']}_PCT" - return v + @model_validator(mode="after") + def set_new_column_name_if_not_set(self) -> "PercentageStep": + if self.new_column_name is None: + self.new_column_name = f"{self.column}_PCT" + return self diff --git a/server/src/weaverbird/pipeline/steps/rename.py b/server/src/weaverbird/pipeline/steps/rename.py index 8f11f88e09..48fd432f15 100644 --- a/server/src/weaverbird/pipeline/steps/rename.py +++ b/server/src/weaverbird/pipeline/steps/rename.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import root_validator +from pydantic import model_validator from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.steps.utils.render_variables import StepWithVariablesMixin @@ -11,7 +11,8 @@ class RenameStep(BaseStep): name: Literal["rename"] = "rename" to_rename: list[tuple[str, str]] - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def handle_legacy_syntax(cls, values): if "oldname" in values and "newname" in values: values["to_rename"] = [(values.pop("oldname"), values.pop("newname"))] diff --git a/server/src/weaverbird/pipeline/steps/replace.py b/server/src/weaverbird/pipeline/steps/replace.py index 51e6b4544f..b74f4c588c 100644 --- a/server/src/weaverbird/pipeline/steps/replace.py +++ b/server/src/weaverbird/pipeline/steps/replace.py @@ -10,7 +10,7 @@ class ReplaceStep(BaseStep): name: Literal["replace"] = "replace" search_column: ColumnName - to_replace: list[tuple[Any, Any]] = Field(min_items=1) + to_replace: list[tuple[Any, Any]] = Field(min_length=1) class ReplaceStepWithVariable(ReplaceStep, StepWithVariablesMixin): diff --git a/server/src/weaverbird/pipeline/steps/rollup.py b/server/src/weaverbird/pipeline/steps/rollup.py index da482c8fc6..c228d274c0 100644 --- a/server/src/weaverbird/pipeline/steps/rollup.py +++ b/server/src/weaverbird/pipeline/steps/rollup.py @@ -14,13 +14,13 @@ class RollupStep(BaseStep): # The list of columnns to aggregate, with related aggregation function to use: aggregations: Sequence[Aggregation] # Groupby columns if rollup has to be performed by groups: - groupby: list[ColumnName] | None + groupby: list[ColumnName] | None = None # To give a custom name to the output label column: - label_col: ColumnName | None + label_col: ColumnName | None = None # To give a custom name to the output level column: - level_col: ColumnName | None + level_col: ColumnName | None = None # To give a custom name to the output parent column: - parent_label_col: ColumnName | None + parent_label_col: ColumnName | None = None class RollupStepWithVariable(RollupStep, StepWithVariablesMixin): diff --git a/server/src/weaverbird/pipeline/steps/select.py b/server/src/weaverbird/pipeline/steps/select.py index f8f0d7739e..3bf91226e9 100644 --- a/server/src/weaverbird/pipeline/steps/select.py +++ b/server/src/weaverbird/pipeline/steps/select.py @@ -8,4 +8,4 @@ class SelectStep(BaseStep): name: Literal["select"] = "select" - columns: list[ColumnName] = Field(min_items=1) + columns: list[ColumnName] = Field(min_length=1) diff --git a/server/src/weaverbird/pipeline/steps/simplify.py b/server/src/weaverbird/pipeline/steps/simplify.py index 3e4b12281c..a134ea3b0f 100644 --- a/server/src/weaverbird/pipeline/steps/simplify.py +++ b/server/src/weaverbird/pipeline/steps/simplify.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import validator +from pydantic import field_validator from weaverbird.pipeline.steps.utils.base import BaseStep @@ -10,7 +10,8 @@ class SimplifyStep(BaseStep): # All parts of the simplified geometry will be no more than this distance from the original tolerance: float = 1.0 - @validator("tolerance") + @field_validator("tolerance") + @classmethod def _tolerance_validator(cls, value: float) -> float: assert value > 0, "tolerance must be strictly positive" return value diff --git a/server/src/weaverbird/pipeline/steps/statistics.py b/server/src/weaverbird/pipeline/steps/statistics.py index 1beadb3979..abcab7ea29 100644 --- a/server/src/weaverbird/pipeline/steps/statistics.py +++ b/server/src/weaverbird/pipeline/steps/statistics.py @@ -11,7 +11,7 @@ class Quantile(BaseModel): - label: str | None + label: str | None = None nth: int order: int diff --git a/server/src/weaverbird/pipeline/steps/substring.py b/server/src/weaverbird/pipeline/steps/substring.py index bfdb4338b3..cf7cc0c80a 100644 --- a/server/src/weaverbird/pipeline/steps/substring.py +++ b/server/src/weaverbird/pipeline/steps/substring.py @@ -7,6 +7,6 @@ class SubstringStep(BaseStep): name: Literal["substring"] = "substring" column: ColumnName - new_column_name: ColumnName | None + new_column_name: ColumnName | None = None start_index: int end_index: int diff --git a/server/src/weaverbird/pipeline/steps/text.py b/server/src/weaverbird/pipeline/steps/text.py index bed8ad40f6..6474cb640a 100644 --- a/server/src/weaverbird/pipeline/steps/text.py +++ b/server/src/weaverbird/pipeline/steps/text.py @@ -2,9 +2,9 @@ from typing import Literal from zoneinfo import ZoneInfo -from pydantic import BaseConfig, Extra, validator +from pydantic import field_validator -from weaverbird.pipeline.steps.utils.base import BaseStep, to_camelcase +from weaverbird.pipeline.steps.utils.base import BaseStep from weaverbird.pipeline.steps.utils.render_variables import StepWithVariablesMixin from weaverbird.pipeline.types import ColumnName @@ -14,13 +14,8 @@ class TextStep(BaseStep): text: datetime | int | float | bool | str new_column: ColumnName - class Config(BaseConfig): - allow_population_by_field_name = True - extra = Extra.forbid - alias_generator = to_camelcase - smart_union = True - - @validator("text") + @field_validator("text") + @classmethod def _text_validator(cls, value): if isinstance(value, datetime) and value.tzinfo is not None: return value.astimezone(ZoneInfo("UTC")).replace(tzinfo=None) diff --git a/server/src/weaverbird/pipeline/steps/todate.py b/server/src/weaverbird/pipeline/steps/todate.py index b51350bac4..bc6cb9d837 100644 --- a/server/src/weaverbird/pipeline/steps/todate.py +++ b/server/src/weaverbird/pipeline/steps/todate.py @@ -7,4 +7,4 @@ class ToDateStep(BaseStep): name: Literal["todate"] = "todate" column: ColumnName - format: str | None + format: str | None = None diff --git a/server/src/weaverbird/pipeline/steps/totals.py b/server/src/weaverbird/pipeline/steps/totals.py index ed6ab5e6a1..d3d83dbbca 100644 --- a/server/src/weaverbird/pipeline/steps/totals.py +++ b/server/src/weaverbird/pipeline/steps/totals.py @@ -1,7 +1,7 @@ from collections.abc import Sequence from typing import Literal -from pydantic import Field, validator +from pydantic import Field, field_validator from weaverbird.pipeline.steps.utils.base import BaseModel, BaseStep from weaverbird.pipeline.steps.utils.render_variables import StepWithVariablesMixin @@ -19,9 +19,10 @@ class TotalsStep(BaseStep): name: Literal["totals"] = "totals" total_dimensions: list[TotalDimension] aggregations: Sequence[Aggregation] - groups: list[ColumnName] = Field(min_items=0, default_factory=list) + groups: list[ColumnName] = Field(min_length=0, default_factory=list) - @validator("aggregations") + @field_validator("aggregations") + @classmethod def aggregation_must_not_be_empty(cls, value): if len(value) < 1: raise ValueError("aggregations must contain at least one item") diff --git a/server/src/weaverbird/pipeline/steps/utils/base.py b/server/src/weaverbird/pipeline/steps/utils/base.py index fc14adefc9..c945b6e806 100644 --- a/server/src/weaverbird/pipeline/steps/utils/base.py +++ b/server/src/weaverbird/pipeline/steps/utils/base.py @@ -1,5 +1,7 @@ -from pydantic import BaseConfig, Extra +from typing import Any + from pydantic import BaseModel as PydanticBaseModel +from pydantic import ConfigDict def to_camelcase(string: str) -> str: @@ -8,15 +10,15 @@ def to_camelcase(string: str) -> str: class BaseModel(PydanticBaseModel): - class Config(BaseConfig): - allow_population_by_field_name = True - extra = Extra.forbid - alias_generator = to_camelcase + model_config = ConfigDict(populate_by_name=True, extra="forbid", alias_generator=to_camelcase) class BaseStep(BaseModel): name: str + def model_dump(self, *, exclude_none: bool = True, **kwargs) -> dict[str, Any]: + return super().model_dump(exclude_none=exclude_none, **kwargs) + # None values are excluded, to avoid triggering validations error in front-ends - def dict(self, *, exclude_none: bool = True, **kwargs) -> dict: - return super().dict(exclude_none=True, **kwargs) + def dict(self, *, exclude_none: bool = True, **kwargs) -> dict[str, Any]: + return self.model_dump(exclude_none=exclude_none, **kwargs) diff --git a/server/src/weaverbird/pipeline/steps/utils/combination.py b/server/src/weaverbird/pipeline/steps/utils/combination.py index 3e8e8a6f80..06b55fb96d 100644 --- a/server/src/weaverbird/pipeline/steps/utils/combination.py +++ b/server/src/weaverbird/pipeline/steps/utils/combination.py @@ -1,7 +1,10 @@ -from collections.abc import Awaitable, Callable -from typing import Literal +from collections.abc import Awaitable, Callable, Iterable +from typing import TYPE_CHECKING, Annotated, Literal -from pydantic import BaseModel +from pydantic import BaseModel, BeforeValidator, TypeAdapter + +if TYPE_CHECKING: + from weaverbird.pipeline.pipeline import PipelineStep, PipelineStepWithRefs class Reference(BaseModel): @@ -14,8 +17,87 @@ def __hash__(self): return hash(f"__ref__{self.uid}") -PipelineOrDomainName = list[dict] | str # can be either a domain name or a complete pipeline +# FIXME: Required because of https://github.com/pydantic/pydantic/issues/7487. +# Repro case: +# { +# "name": "append", +# "pipelines": [ +# [ +# {"name": "domain", "domain": "styles"}, +# { +# "name": "join", +# "type": "inner", +# "right_pipeline": [ +# {"name": "domain", "domain": "beers"}, +# {"name": "uppercase", "column": "a"}, +# {"name": "uppercase", "column": "b"}, +# ], +# "on": [("style", "name")], +# }, +# ], +# [ +# {"name": "domain", "domain": "beers"}, +# {"name": "uppercase", "column": "a"}, +# ], +# ], +# } + + +def _ensure_is_pipeline_step( + v: str | list[dict] | list["PipelineStep"], +) -> str | list["PipelineStep"]: + from weaverbird.pipeline.pipeline import PipelineStep + + adapter = TypeAdapter(PipelineStep) + if isinstance(v, str): + return v + + def iter_() -> Iterable["PipelineStep"]: + for elem in v: + if isinstance(elem, dict): + yield adapter.validate_python(elem) + else: + yield elem + + out = list(iter_()) + return out + + +# can be either a domain name or a complete pipeline +PipelineOrDomainName = Annotated[ + str | list["PipelineStep"], BeforeValidator(_ensure_is_pipeline_step) +] + + +def _ensure_is_pipeline_step_with_ref( + v: str | list[dict] | list["PipelineStep | PipelineStepWithRefs"], +) -> str | list["PipelineStep"]: + from weaverbird.pipeline.pipeline import PipelineStep, PipelineStepWithRefs + + adapter = TypeAdapter(PipelineStepWithRefs | PipelineStep) + if isinstance(v, str): + return v + + def iter_() -> Iterable["PipelineStepWithRefs | PipelineStep"]: + for elem in v: + if isinstance(elem, dict): + yield adapter.validate_python(elem) + else: + yield elem + + out = list(iter_()) + return out + + +# can be either a domain name or a complete pipeline +PipelineWithRefsOrDomainName = Annotated[ + str | list["PipelineStepWithRefs | PipelineStep"], + BeforeValidator(_ensure_is_pipeline_step_with_ref), +] + + PipelineOrDomainNameOrReference = PipelineOrDomainName | Reference +PipelineWithRefsOrDomainNameOrReference = PipelineWithRefsOrDomainName | Reference # A reference returning None means that it should be skipped ReferenceResolver = Callable[[Reference], Awaitable[PipelineOrDomainName | None]] @@ -25,18 +107,17 @@ async def resolve_if_reference( reference_resolver: ReferenceResolver, pipeline_or_domain_name_or_ref: PipelineOrDomainNameOrReference, ) -> PipelineOrDomainName | None: - from weaverbird.pipeline.references import PipelineWithRefs + from weaverbird.pipeline.pipeline import PipelineWithRefs, ReferenceUnresolved if isinstance(pipeline_or_domain_name_or_ref, Reference): pipeline_or_domain_name = await reference_resolver(pipeline_or_domain_name_or_ref) if isinstance(pipeline_or_domain_name, list): # Recursively resolve any reference in sub-pipelines pipeline = PipelineWithRefs(steps=pipeline_or_domain_name) - from weaverbird.pipeline.references import ReferenceUnresolved try: pipeline_without_refs = await pipeline.resolve_references(reference_resolver) - return pipeline_without_refs.dict()["steps"] + return pipeline_without_refs.model_dump()["steps"] except ReferenceUnresolved: return None # skip else: diff --git a/server/src/weaverbird/pipeline/steps/utils/render_variables.py b/server/src/weaverbird/pipeline/steps/utils/render_variables.py index 473e81bd3d..c9a48dd365 100644 --- a/server/src/weaverbird/pipeline/steps/utils/render_variables.py +++ b/server/src/weaverbird/pipeline/steps/utils/render_variables.py @@ -7,6 +7,6 @@ class StepWithVariablesMixin: def render(self, variables: dict[str, Any], renderer: Callable[[Any, Any], Any]) -> BaseStep: rendered_class = self.__class__.__bases__[0] - step_as_dict = self.dict() # type: ignore + step_as_dict = self.model_dump() # type: ignore rendered_dict = renderer(step_as_dict, variables) return rendered_class(**rendered_dict) diff --git a/server/src/weaverbird/pipeline/steps/waterfall.py b/server/src/weaverbird/pipeline/steps/waterfall.py index a276989b10..bd45c14ed2 100644 --- a/server/src/weaverbird/pipeline/steps/waterfall.py +++ b/server/src/weaverbird/pipeline/steps/waterfall.py @@ -20,7 +20,7 @@ class WaterfallStep(BaseStep): labelsColumn: ColumnName sortBy: Literal["value", "label"] order: Literal["desc", "asc"] - parentsColumn: ColumnName | None + parentsColumn: ColumnName | None = None groupby: list[ColumnName] = [] backfill: bool = True diff --git a/server/tests/pipeline/test_references.py b/server/tests/pipeline/test_references.py index 0a5bf41fd6..d0a2aab8c9 100644 --- a/server/tests/pipeline/test_references.py +++ b/server/tests/pipeline/test_references.py @@ -1,14 +1,19 @@ import pytest -from weaverbird.pipeline.pipeline import Pipeline, PipelineWithVariables -from weaverbird.pipeline.references import PipelineWithRefs, ReferenceUnresolved +from weaverbird.pipeline.pipeline import ( + Pipeline, + PipelineWithRefs, + PipelineWithVariables, + ReferenceUnresolved, +) from weaverbird.pipeline.steps import ( AppendStepWithRefs, DomainStep, DomainStepWithRef, - JoinStep, JoinStepWithRef, TextStep, ) +from weaverbird.pipeline.steps.append import AppendStepWithVariable +from weaverbird.pipeline.steps.join import JoinStepWithVariable from weaverbird.pipeline.steps.utils.combination import Reference PIPELINES_LIBRARY: dict[str, list[dict]] = { @@ -88,10 +93,11 @@ async def test_resolve_references_join(): TextStep(new_column="text", text="Lorem ipsum"), ] ) - assert await pipeline_with_refs.resolve_references(reference_resolver) == PipelineWithVariables( + + expected = PipelineWithVariables( steps=[ DomainStep(domain="source"), - JoinStep( + JoinStepWithVariable( on=[("key_left", "key_right")], right_pipeline=PIPELINES_LIBRARY["other_pipeline"], type="left", @@ -100,6 +106,8 @@ async def test_resolve_references_join(): ] ) + assert await pipeline_with_refs.resolve_references(reference_resolver) == expected + @pytest.mark.asyncio async def test_resolve_references_append(): @@ -115,18 +123,16 @@ async def test_resolve_references_append(): ] ) - assert await pipeline_with_refs.resolve_references(reference_resolver) == PipelineWithVariables( + expected = PipelineWithVariables( steps=[ DomainStep(domain="source"), - AppendStepWithRefs( - pipelines=[ - PIPELINES_LIBRARY["other_pipeline"], - ] - ), + AppendStepWithVariable(pipelines=[PIPELINES_LIBRARY["other_pipeline"]]), TextStep(new_column="text", text="Lorem ipsum"), ] ) + assert await pipeline_with_refs.resolve_references(reference_resolver) == expected + @pytest.mark.asyncio async def test_resolve_references_unresolved_append(): @@ -147,17 +153,14 @@ async def test_resolve_references_unresolved_append(): ] ) - assert await pipeline_with_refs.resolve_references(reference_resolver) == PipelineWithVariables( + expected = PipelineWithVariables( steps=[ DomainStep(domain="source"), - AppendStepWithRefs( - pipelines=[ - PIPELINES_LIBRARY["other_pipeline"], - ] - ), + AppendStepWithVariable(pipelines=[PIPELINES_LIBRARY["other_pipeline"]]), TextStep(new_column="text", text="Lorem ipsum"), ] ) + assert await pipeline_with_refs.resolve_references(reference_resolver) == expected @pytest.mark.asyncio @@ -219,18 +222,16 @@ async def test_resolve_references_unresolved_append_subpipeline_error(): ] ) - assert await pipeline_with_refs.resolve_references(reference_resolver) == PipelineWithVariables( + expected = PipelineWithVariables( steps=[ DomainStep(domain="source"), - AppendStepWithRefs( - pipelines=[ - PIPELINES_LIBRARY["other_pipeline"], - ] - ), + AppendStepWithVariable(pipelines=[PIPELINES_LIBRARY["other_pipeline"]]), TextStep(new_column="text", text="Lorem ipsum"), ] ) + assert await pipeline_with_refs.resolve_references(reference_resolver) == expected + @pytest.mark.asyncio async def test_resolve_references_unresolved_join():