From 65a9774d9ef64eb5c873be1562a32051b18b1233 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 10:21:51 +0200 Subject: [PATCH 01/27] chore: Bump pydantic to ^2.3.0 Signed-off-by: Luka Peschke --- poetry.lock | 188 +++++++++++++++++++++++++++++++++++++------------ pyproject.toml | 2 +- 2 files changed, 143 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2c3362959..127bf5a9d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -185,6 +185,17 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "annotated-types" +version = "0.5.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.7" +files = [ + {file = "annotated_types-0.5.0-py3-none-any.whl", hash = "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd"}, + {file = "annotated_types-0.5.0.tar.gz", hash = "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802"}, +] + [[package]] name = "asn1crypto" version = "1.5.1" @@ -1605,7 +1616,7 @@ grpc = ["grpcio (>=1.0.0,<2.0.0dev)"] name = "grpcio" version = "1.53.0" description = "HTTP/2-based RPC framework" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "grpcio-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f"}, @@ -2790,55 +2801,140 @@ files = [ [[package]] name = "pydantic" -version = "1.10.12" -description = "Data validation and settings management using python type hints" +version = "2.3.0" +description = "Data validation using Python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, - {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, - {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, - {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, - {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, - {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, - {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, - {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, - {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, - {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, + {file = "pydantic-2.3.0-py3-none-any.whl", hash = "sha256:45b5e446c6dfaad9444819a293b921a40e1db1aa61ea08aede0522529ce90e81"}, + {file = "pydantic-2.3.0.tar.gz", hash = "sha256:1607cc106602284cd4a00882986570472f193fde9cb1259bceeaedb26aa79a6d"}, ] [package.dependencies] -typing-extensions = ">=4.2.0" +annotated-types = ">=0.4.0" +pydantic-core = "2.6.3" +typing-extensions = ">=4.6.1" [package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.6.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic_core-2.6.3-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:1a0ddaa723c48af27d19f27f1c73bdc615c73686d763388c8683fe34ae777bad"}, + {file = "pydantic_core-2.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5cfde4fab34dd1e3a3f7f3db38182ab6c95e4ea91cf322242ee0be5c2f7e3d2f"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5493a7027bfc6b108e17c3383959485087d5942e87eb62bbac69829eae9bc1f7"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:84e87c16f582f5c753b7f39a71bd6647255512191be2d2dbf49458c4ef024588"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:522a9c4a4d1924facce7270c84b5134c5cabcb01513213662a2e89cf28c1d309"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaafc776e5edc72b3cad1ccedb5fd869cc5c9a591f1213aa9eba31a781be9ac1"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a750a83b2728299ca12e003d73d1264ad0440f60f4fc9cee54acc489249b728"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e8b374ef41ad5c461efb7a140ce4730661aadf85958b5c6a3e9cf4e040ff4bb"}, + {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b594b64e8568cf09ee5c9501ede37066b9fc41d83d58f55b9952e32141256acd"}, + {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2a20c533cb80466c1d42a43a4521669ccad7cf2967830ac62c2c2f9cece63e7e"}, + {file = "pydantic_core-2.6.3-cp310-none-win32.whl", hash = "sha256:04fe5c0a43dec39aedba0ec9579001061d4653a9b53a1366b113aca4a3c05ca7"}, + {file = "pydantic_core-2.6.3-cp310-none-win_amd64.whl", hash = "sha256:6bf7d610ac8f0065a286002a23bcce241ea8248c71988bda538edcc90e0c39ad"}, + {file = "pydantic_core-2.6.3-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:6bcc1ad776fffe25ea5c187a028991c031a00ff92d012ca1cc4714087e575973"}, + {file = "pydantic_core-2.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:df14f6332834444b4a37685810216cc8fe1fe91f447332cd56294c984ecbff1c"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b7486d85293f7f0bbc39b34e1d8aa26210b450bbd3d245ec3d732864009819"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a892b5b1871b301ce20d40b037ffbe33d1407a39639c2b05356acfef5536d26a"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:883daa467865e5766931e07eb20f3e8152324f0adf52658f4d302242c12e2c32"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4eb77df2964b64ba190eee00b2312a1fd7a862af8918ec70fc2d6308f76ac64"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce8c84051fa292a5dc54018a40e2a1926fd17980a9422c973e3ebea017aa8da"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22134a4453bd59b7d1e895c455fe277af9d9d9fbbcb9dc3f4a97b8693e7e2c9b"}, + {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:02e1c385095efbd997311d85c6021d32369675c09bcbfff3b69d84e59dc103f6"}, + {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d79f1f2f7ebdb9b741296b69049ff44aedd95976bfee38eb4848820628a99b50"}, + {file = "pydantic_core-2.6.3-cp311-none-win32.whl", hash = "sha256:430ddd965ffd068dd70ef4e4d74f2c489c3a313adc28e829dd7262cc0d2dd1e8"}, + {file = "pydantic_core-2.6.3-cp311-none-win_amd64.whl", hash = "sha256:84f8bb34fe76c68c9d96b77c60cef093f5e660ef8e43a6cbfcd991017d375950"}, + {file = "pydantic_core-2.6.3-cp311-none-win_arm64.whl", hash = "sha256:5a2a3c9ef904dcdadb550eedf3291ec3f229431b0084666e2c2aa8ff99a103a2"}, + {file = "pydantic_core-2.6.3-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:8421cf496e746cf8d6b677502ed9a0d1e4e956586cd8b221e1312e0841c002d5"}, + {file = "pydantic_core-2.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bb128c30cf1df0ab78166ded1ecf876620fb9aac84d2413e8ea1594b588c735d"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37a822f630712817b6ecc09ccc378192ef5ff12e2c9bae97eb5968a6cdf3b862"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:240a015102a0c0cc8114f1cba6444499a8a4d0333e178bc504a5c2196defd456"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f90e5e3afb11268628c89f378f7a1ea3f2fe502a28af4192e30a6cdea1e7d5e"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:340e96c08de1069f3d022a85c2a8c63529fd88709468373b418f4cf2c949fb0e"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1480fa4682e8202b560dcdc9eeec1005f62a15742b813c88cdc01d44e85308e5"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f14546403c2a1d11a130b537dda28f07eb6c1805a43dae4617448074fd49c282"}, + {file = "pydantic_core-2.6.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a87c54e72aa2ef30189dc74427421e074ab4561cf2bf314589f6af5b37f45e6d"}, + {file = "pydantic_core-2.6.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f93255b3e4d64785554e544c1c76cd32f4a354fa79e2eeca5d16ac2e7fdd57aa"}, + {file = "pydantic_core-2.6.3-cp312-none-win32.whl", hash = "sha256:f70dc00a91311a1aea124e5f64569ea44c011b58433981313202c46bccbec0e1"}, + {file = "pydantic_core-2.6.3-cp312-none-win_amd64.whl", hash = "sha256:23470a23614c701b37252618e7851e595060a96a23016f9a084f3f92f5ed5881"}, + {file = "pydantic_core-2.6.3-cp312-none-win_arm64.whl", hash = "sha256:1ac1750df1b4339b543531ce793b8fd5c16660a95d13aecaab26b44ce11775e9"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:a53e3195f134bde03620d87a7e2b2f2046e0e5a8195e66d0f244d6d5b2f6d31b"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:f2969e8f72c6236c51f91fbb79c33821d12a811e2a94b7aa59c65f8dbdfad34a"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:672174480a85386dd2e681cadd7d951471ad0bb028ed744c895f11f9d51b9ebe"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:002d0ea50e17ed982c2d65b480bd975fc41086a5a2f9c924ef8fc54419d1dea3"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ccc13afee44b9006a73d2046068d4df96dc5b333bf3509d9a06d1b42db6d8bf"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:439a0de139556745ae53f9cc9668c6c2053444af940d3ef3ecad95b079bc9987"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d63b7545d489422d417a0cae6f9898618669608750fc5e62156957e609e728a5"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b44c42edc07a50a081672e25dfe6022554b47f91e793066a7b601ca290f71e42"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1c721bfc575d57305dd922e6a40a8fe3f762905851d694245807a351ad255c58"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5e4a2cf8c4543f37f5dc881de6c190de08096c53986381daebb56a355be5dfe6"}, + {file = "pydantic_core-2.6.3-cp37-none-win32.whl", hash = "sha256:d9b4916b21931b08096efed090327f8fe78e09ae8f5ad44e07f5c72a7eedb51b"}, + {file = "pydantic_core-2.6.3-cp37-none-win_amd64.whl", hash = "sha256:a8acc9dedd304da161eb071cc7ff1326aa5b66aadec9622b2574ad3ffe225525"}, + {file = "pydantic_core-2.6.3-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:5e9c068f36b9f396399d43bfb6defd4cc99c36215f6ff33ac8b9c14ba15bdf6b"}, + {file = "pydantic_core-2.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e61eae9b31799c32c5f9b7be906be3380e699e74b2db26c227c50a5fc7988698"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85463560c67fc65cd86153a4975d0b720b6d7725cf7ee0b2d291288433fc21b"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9616567800bdc83ce136e5847d41008a1d602213d024207b0ff6cab6753fe645"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e9b65a55bbabda7fccd3500192a79f6e474d8d36e78d1685496aad5f9dbd92c"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f468d520f47807d1eb5d27648393519655eadc578d5dd862d06873cce04c4d1b"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9680dd23055dd874173a3a63a44e7f5a13885a4cfd7e84814be71be24fba83db"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a718d56c4d55efcfc63f680f207c9f19c8376e5a8a67773535e6f7e80e93170"}, + {file = "pydantic_core-2.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8ecbac050856eb6c3046dea655b39216597e373aa8e50e134c0e202f9c47efec"}, + {file = "pydantic_core-2.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:788be9844a6e5c4612b74512a76b2153f1877cd845410d756841f6c3420230eb"}, + {file = "pydantic_core-2.6.3-cp38-none-win32.whl", hash = "sha256:07a1aec07333bf5adebd8264047d3dc518563d92aca6f2f5b36f505132399efc"}, + {file = "pydantic_core-2.6.3-cp38-none-win_amd64.whl", hash = "sha256:621afe25cc2b3c4ba05fff53525156d5100eb35c6e5a7cf31d66cc9e1963e378"}, + {file = "pydantic_core-2.6.3-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:813aab5bfb19c98ae370952b6f7190f1e28e565909bfc219a0909db168783465"}, + {file = "pydantic_core-2.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:50555ba3cb58f9861b7a48c493636b996a617db1a72c18da4d7f16d7b1b9952b"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e20f8baedd7d987bd3f8005c146e6bcbda7cdeefc36fad50c66adb2dd2da48"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0a5d7edb76c1c57b95df719af703e796fc8e796447a1da939f97bfa8a918d60"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f06e21ad0b504658a3a9edd3d8530e8cea5723f6ea5d280e8db8efc625b47e49"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea053cefa008fda40f92aab937fb9f183cf8752e41dbc7bc68917884454c6362"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:171a4718860790f66d6c2eda1d95dd1edf64f864d2e9f9115840840cf5b5713f"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ed7ceca6aba5331ece96c0e328cd52f0dcf942b8895a1ed2642de50800b79d3"}, + {file = "pydantic_core-2.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:acafc4368b289a9f291e204d2c4c75908557d4f36bd3ae937914d4529bf62a76"}, + {file = "pydantic_core-2.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1aa712ba150d5105814e53cb141412217146fedc22621e9acff9236d77d2a5ef"}, + {file = "pydantic_core-2.6.3-cp39-none-win32.whl", hash = "sha256:44b4f937b992394a2e81a5c5ce716f3dcc1237281e81b80c748b2da6dd5cf29a"}, + {file = "pydantic_core-2.6.3-cp39-none-win_amd64.whl", hash = "sha256:9b33bf9658cb29ac1a517c11e865112316d09687d767d7a0e4a63d5c640d1b17"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d7050899026e708fb185e174c63ebc2c4ee7a0c17b0a96ebc50e1f76a231c057"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:99faba727727b2e59129c59542284efebbddade4f0ae6a29c8b8d3e1f437beb7"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fa159b902d22b283b680ef52b532b29554ea2a7fc39bf354064751369e9dbd7"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:046af9cfb5384f3684eeb3f58a48698ddab8dd870b4b3f67f825353a14441418"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:930bfe73e665ebce3f0da2c6d64455098aaa67e1a00323c74dc752627879fc67"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:85cc4d105747d2aa3c5cf3e37dac50141bff779545ba59a095f4a96b0a460e70"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b25afe9d5c4f60dcbbe2b277a79be114e2e65a16598db8abee2a2dcde24f162b"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e49ce7dc9f925e1fb010fc3d555250139df61fa6e5a0a95ce356329602c11ea9"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:2dd50d6a1aef0426a1d0199190c6c43ec89812b1f409e7fe44cb0fbf6dfa733c"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6595b0d8c8711e8e1dc389d52648b923b809f68ac1c6f0baa525c6440aa0daa"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef724a059396751aef71e847178d66ad7fc3fc969a1a40c29f5aac1aa5f8784"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c8945a105f1589ce8a693753b908815e0748f6279959a4530f6742e1994dcb6"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c8c6660089a25d45333cb9db56bb9e347241a6d7509838dbbd1931d0e19dbc7f"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:692b4ff5c4e828a38716cfa92667661a39886e71136c97b7dac26edef18767f7"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f1a5d8f18877474c80b7711d870db0eeef9442691fcdb00adabfc97e183ee0b0"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3796a6152c545339d3b1652183e786df648ecdf7c4f9347e1d30e6750907f5bb"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b962700962f6e7a6bd77e5f37320cabac24b4c0f76afeac05e9f93cf0c620014"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56ea80269077003eaa59723bac1d8bacd2cd15ae30456f2890811efc1e3d4413"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c0ebbebae71ed1e385f7dfd9b74c1cff09fed24a6df43d326dd7f12339ec34"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:252851b38bad3bfda47b104ffd077d4f9604a10cb06fe09d020016a25107bf98"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6656a0ae383d8cd7cc94e91de4e526407b3726049ce8d7939049cbfa426518c8"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9140ded382a5b04a1c030b593ed9bf3088243a0a8b7fa9f071a5736498c5483"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d38bbcef58220f9c81e42c255ef0bf99735d8f11edef69ab0b499da77105158a"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:c9d469204abcca28926cbc28ce98f28e50e488767b084fb3fbdf21af11d3de26"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:48c1ed8b02ffea4d5c9c220eda27af02b8149fe58526359b3c07eb391cb353a2"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2b1bfed698fa410ab81982f681f5b1996d3d994ae8073286515ac4d165c2e7"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf9d42a71a4d7a7c1f14f629e5c30eac451a6fc81827d2beefd57d014c006c4a"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4292ca56751aebbe63a84bbfc3b5717abb09b14d4b4442cc43fd7c49a1529efd"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7dc2ce039c7290b4ef64334ec7e6ca6494de6eecc81e21cb4f73b9b39991408c"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:615a31b1629e12445c0e9fc8339b41aaa6cc60bd53bf802d5fe3d2c0cda2ae8d"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1fa1f6312fb84e8c281f32b39affe81984ccd484da6e9d65b3d18c202c666149"}, + {file = "pydantic_core-2.6.3.tar.gz", hash = "sha256:1508f37ba9e3ddc0189e6ff4e2228bd2d3c3a4641cbe8c07177162f76ed696c7"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyflakes" @@ -4083,13 +4179,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.5.0" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] @@ -4459,4 +4555,4 @@ toucan-toco = ["toucan-client"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "08eb6477169a0656f4f29ee5a7dc3a6f2c1c7b8ab174aa97f9f9bc023787ff32" +content-hash = "eacc7650ad5a673f87d078b928ad25c5eb12dee8c627c141cfeb276867e4ef14" diff --git a/pyproject.toml b/pyproject.toml index 72dfe7466..17031e06d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ Authlib = "^1.0.1" cached-property = "^1.5.2" Jinja2 = "^3.0.3" jq = "^1.2.2" -pydantic = "^1.9.1" +pydantic = "^2.3.0" requests = "^2.28.0" tenacity = "^8.0.1" toucan-data-sdk = "^7.6.0" From e178f74fb6360c87114cd43faeb41707aceaee1d Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 10:23:28 +0200 Subject: [PATCH 02/27] chore: run bump-pydantic Signed-off-by: Luka Peschke --- .../anaplan/anaplan_connector.py | 9 ++++--- .../awsathena/awsathena_connector.py | 14 +++++------ .../azure_mssql/azure_mssql_connector.py | 5 ++-- .../clickhouse/clickhouse_connector.py | 9 ++++--- .../databricks/databricks_connector.py | 5 ++-- .../facebook_ads/facebook_ads_connector.py | 2 +- toucan_connectors/github/github_connector.py | 2 +- .../google_adwords_connector.py | 4 ++- .../google_analytics_connector.py | 17 +++---------- .../google_big_query_connector.py | 9 +++---- .../google_cloud_mysql_connector.py | 5 ++-- toucan_connectors/google_credentials.py | 5 ++-- .../google_sheets/google_sheets_connector.py | 2 ++ .../google_sheets_2_connector.py | 4 ++- .../http_api/http_api_connector.py | 2 ++ .../hubspot/hubspot_connector.py | 2 +- .../hubspot_private_app/hubspot_connector.py | 12 +++------ .../linkedinads/linkedinads_connector.py | 4 ++- toucan_connectors/mongo/mongo_connector.py | 8 +++--- toucan_connectors/mssql/mssql_connector.py | 7 +++--- .../mssql_TLSv1_0/mssql_connector.py | 7 +++--- toucan_connectors/mysql/mysql_connector.py | 14 ++++++----- toucan_connectors/odbc/odbc_connector.py | 5 ++-- .../one_drive/one_drive_connector.py | 4 +-- .../oracle_sql/oracle_sql_connector.py | 7 +++--- toucan_connectors/pagination.py | 6 ++--- .../peakina/peakina_connector.py | 4 +-- .../postgres/postgresql_connector.py | 7 +++--- .../redshift/redshift_database_connector.py | 10 +++++--- .../salesforce/salesforce_connector.py | 2 +- .../sap_hana/sap_hana_connector.py | 5 ++-- .../snowflake/snowflake_connector.py | 2 ++ toucan_connectors/snowflake_common.py | 5 ++-- toucan_connectors/soap/soap_connector.py | 2 ++ toucan_connectors/toucan_connector.py | 25 +++++++++---------- 35 files changed, 126 insertions(+), 106 deletions(-) diff --git a/toucan_connectors/anaplan/anaplan_connector.py b/toucan_connectors/anaplan/anaplan_connector.py index 84f0cbea8..11f5d14f4 100644 --- a/toucan_connectors/anaplan/anaplan_connector.py +++ b/toucan_connectors/anaplan/anaplan_connector.py @@ -4,11 +4,12 @@ import pandas as pd import pydantic import requests -from pydantic import Field, constr, create_model +from pydantic import StringConstraints, Field, create_model from pydantic.types import SecretStr from toucan_connectors.common import ConnectorStatus from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from typing_extensions import Annotated _ID_SEPARATOR = ' - ' @@ -22,8 +23,8 @@ def _format_name_and_id(obj: Dict[str, str]) -> str: class AnaplanDataSource(ToucanDataSource): - model_id: constr(min_length=1) = Field(..., description='The model you want to query') - view_id: constr(min_length=1) = Field(..., description='The view you want to query') + model_id: Annotated[str, StringConstraints(min_length=1)] = Field(..., description='The model you want to query') + view_id: Annotated[str, StringConstraints(min_length=1)] = Field(..., description='The view you want to query') workspace_id: str = Field(..., description='The ID of the workspace you want to query') @pydantic.validator('model_id', 'view_id', 'workspace_id') @@ -90,7 +91,7 @@ class AnaplanAuthError(AnaplanError): class AnaplanConnector(ToucanConnector): data_source_model: AnaplanDataSource username: str - password: Optional[SecretStr] + password: Optional[SecretStr] = None def _extract_json(self, resp: requests.Response) -> dict: if resp.status_code in (401, 403): diff --git a/toucan_connectors/awsathena/awsathena_connector.py b/toucan_connectors/awsathena/awsathena_connector.py index 66d570cbe..8d00472fd 100644 --- a/toucan_connectors/awsathena/awsathena_connector.py +++ b/toucan_connectors/awsathena/awsathena_connector.py @@ -4,7 +4,7 @@ import boto3 import pandas as pd from cached_property import cached_property_with_ttl -from pydantic import Field, SecretStr, constr, create_model +from pydantic import StringConstraints, ConfigDict, Field, SecretStr, create_model from toucan_connectors.common import ConnectorStatus, apply_query_parameters, sanitize_query from toucan_connectors.pagination import build_pagination_info @@ -18,18 +18,19 @@ ToucanDataSource, strlist_to_enum, ) +from typing_extensions import Annotated class AwsathenaDataSource(ToucanDataSource): name: str = Field(..., description='Your AWS Athena connector name') - database: constr(min_length=1) = Field( + database: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='The name of the database you want to query.' ) table: str = Field( None, **{'ui.hidden': True} ) # To avoid previous config migrations, won't be used language: str = Field('sql', **{'ui.hidden': True}) - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='The SQL query to execute.', widget='sql', @@ -77,10 +78,9 @@ class AwsathenaConnector(ToucanConnector, DiscoverableConnector): aws_access_key_id: str = Field(..., description='Your AWS access key ID') aws_secret_access_key: SecretStr = Field(None, description='Your AWS secret key') region_name: str = Field(..., description='Your AWS region name') - - class Config: - underscore_attrs_are_private = True - keep_untouched = (cached_property_with_ttl,) + # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)) def get_session(self) -> boto3.Session: return boto3.Session( diff --git a/toucan_connectors/azure_mssql/azure_mssql_connector.py b/toucan_connectors/azure_mssql/azure_mssql_connector.py index afc242862..56a94b8c2 100644 --- a/toucan_connectors/azure_mssql/azure_mssql_connector.py +++ b/toucan_connectors/azure_mssql/azure_mssql_connector.py @@ -2,17 +2,18 @@ import pandas as pd import pyodbc -from pydantic import Field, SecretStr, constr +from pydantic import StringConstraints, Field, SecretStr from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from typing_extensions import Annotated CLOUD_HOST = 'database.windows.net' class AzureMSSQLDataSource(ToucanDataSource): database: str = Field(..., description='The name of the database you want to query') - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write your SQL query here', widget='sql' ) diff --git a/toucan_connectors/clickhouse/clickhouse_connector.py b/toucan_connectors/clickhouse/clickhouse_connector.py index efd625b4c..eed1268a7 100644 --- a/toucan_connectors/clickhouse/clickhouse_connector.py +++ b/toucan_connectors/clickhouse/clickhouse_connector.py @@ -2,28 +2,31 @@ from typing import Any, Dict, Type import clickhouse_driver -from pydantic import Field, SecretStr, constr, create_model +from pydantic import StringConstraints, Field, SecretStr, create_model from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from typing_extensions import Annotated class ClickhouseDataSource(ToucanDataSource): database: str = Field(None, description='The name of the database you want to query') - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' 'the "table" parameter above', widget='sql', ) - table: constr(min_length=1) = Field( + table: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='The name of the data table that you want to ' 'get (equivalent to "SELECT * FROM ' 'your_table")', ) + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['ClickhouseDataSource']) -> None: diff --git a/toucan_connectors/databricks/databricks_connector.py b/toucan_connectors/databricks/databricks_connector.py index 7ab0d1576..c0ad93333 100644 --- a/toucan_connectors/databricks/databricks_connector.py +++ b/toucan_connectors/databricks/databricks_connector.py @@ -4,17 +4,18 @@ import pandas as pd import pyodbc import requests -from pydantic import Field, SecretStr, constr +from pydantic import StringConstraints, Field, SecretStr from requests.auth import HTTPBasicAuth from toucan_connectors.common import ClusterStartException, ConnectorStatus, pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from typing_extensions import Annotated logger = logging.getLogger(__name__) class DatabricksDataSource(ToucanDataSource): - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write a query here', widget='sql', diff --git a/toucan_connectors/facebook_ads/facebook_ads_connector.py b/toucan_connectors/facebook_ads/facebook_ads_connector.py index 9bd8b4bbc..d534c45ad 100644 --- a/toucan_connectors/facebook_ads/facebook_ads_connector.py +++ b/toucan_connectors/facebook_ads/facebook_ads_connector.py @@ -78,7 +78,7 @@ def determine_query_params(self) -> Dict[str, str]: class FacebookAdsConnector(ToucanConnector): _auth_flow = 'oauth2' - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None _oauth_trigger = 'instance' data_source_model: FacebookAdsDataSource oauth2_version = Field('1', **{'ui.hidden': True}) diff --git a/toucan_connectors/github/github_connector.py b/toucan_connectors/github/github_connector.py index fd85a411b..951b7f4d6 100644 --- a/toucan_connectors/github/github_connector.py +++ b/toucan_connectors/github/github_connector.py @@ -97,7 +97,7 @@ def get_form(cls, connector: 'GithubConnector', current_config, **kwargs): class GithubConnector(ToucanConnector): _auth_flow = 'oauth2' - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None data_source_model: GithubDataSource _oauth_trigger = 'instance' _oauth2_connector: OAuth2Connector = PrivateAttr() diff --git a/toucan_connectors/google_adwords/google_adwords_connector.py b/toucan_connectors/google_adwords/google_adwords_connector.py index 7b9704036..96388bc75 100644 --- a/toucan_connectors/google_adwords/google_adwords_connector.py +++ b/toucan_connectors/google_adwords/google_adwords_connector.py @@ -59,6 +59,8 @@ class GoogleAdwordsDataSource(ToucanDataSource): description='Max number of rows to extract, for service extraction only', ) + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource']) -> None: @@ -79,7 +81,7 @@ def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource']) class GoogleAdwordsConnector(ToucanConnector): data_source_model: GoogleAdwordsDataSource _auth_flow = 'oauth2' - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None developer_token: str = None client_customer_id: str = None _oauth_trigger = 'instance' diff --git a/toucan_connectors/google_analytics/google_analytics_connector.py b/toucan_connectors/google_analytics/google_analytics_connector.py index ed7d5d8d1..522771e5e 100644 --- a/toucan_connectors/google_analytics/google_analytics_connector.py +++ b/toucan_connectors/google_analytics/google_analytics_connector.py @@ -3,7 +3,7 @@ import pandas as pd from apiclient.discovery import build from oauth2client.service_account import ServiceAccountCredentials -from pydantic import BaseModel, Field +from pydantic import ConfigDict, BaseModel, Field from toucan_connectors.common import nosql_apply_parameters_to_query from toucan_connectors.google_credentials import GoogleCredentials @@ -24,10 +24,7 @@ class DimensionFilter(BaseModel): operator: str expressions: List[str] = None caseSensitive: bool = False - - class Config: - # TODO `not` param is not implemented - extra = 'allow' + model_config = ConfigDict(extra='allow') class DimensionFilterClause(BaseModel): @@ -43,20 +40,14 @@ class DateRange(BaseModel): class Metric(BaseModel): expression: str alias: str = None - - class Config: - # TODO `metricType` param is not implemented - extra = 'allow' + model_config = ConfigDict(extra='allow') class MetricFilter(BaseModel): metricName: str operator: str comparisonValue: str - - class Config: - # TODO `not` param is not implemented - extra = 'allow' + model_config = ConfigDict(extra='allow') class MetricFilterClause(BaseModel): diff --git a/toucan_connectors/google_big_query/google_big_query_connector.py b/toucan_connectors/google_big_query/google_big_query_connector.py index 8404a7522..55f74c77c 100644 --- a/toucan_connectors/google_big_query/google_big_query_connector.py +++ b/toucan_connectors/google_big_query/google_big_query_connector.py @@ -11,7 +11,7 @@ from google.cloud.bigquery.dbapi import _helpers as bigquery_helpers from google.cloud.bigquery.job import QueryJob from google.oauth2.service_account import Credentials -from pydantic import Field, create_model +from pydantic import ConfigDict, Field, create_model from toucan_connectors.common import sanitize_query from toucan_connectors.google_credentials import GoogleCredentials, get_google_oauth2_credentials @@ -103,10 +103,9 @@ class GoogleBigQueryConnector(ToucanConnector, DiscoverableConnector): 'the Google APIs. For more information, see this ' 'documentation', ) - - class Config: - underscore_attrs_are_private = True - keep_untouched = (cached_property,) + # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property,)) @staticmethod def _get_google_credentials(credentials: GoogleCredentials, scopes: List[str]) -> Credentials: diff --git a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py index 77c17b3b6..ffb5c65af 100644 --- a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py +++ b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py @@ -1,14 +1,15 @@ import pandas as pd import pymysql -from pydantic import Field, SecretStr, constr +from pydantic import StringConstraints, Field, SecretStr from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from typing_extensions import Annotated class GoogleCloudMySQLDataSource(ToucanDataSource): database: str = Field(..., description='The name of the database you want to query') - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write your SQL query here', widget='sql' ) diff --git a/toucan_connectors/google_credentials.py b/toucan_connectors/google_credentials.py index acd93d1f0..cc9ee1314 100644 --- a/toucan_connectors/google_credentials.py +++ b/toucan_connectors/google_credentials.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field, HttpUrl, validator +from pydantic import field_validator, BaseModel, Field, HttpUrl CREDENTIALS_INFO_MESSAGE = ( 'This information is provided in your ' @@ -42,7 +42,8 @@ class GoogleCredentials(BaseModel): description=CREDENTIALS_INFO_MESSAGE, ) - @validator('private_key') + @field_validator('private_key') + @classmethod def unescape_break_lines(cls, v): """ `private_key` is a long string like diff --git a/toucan_connectors/google_sheets/google_sheets_connector.py b/toucan_connectors/google_sheets/google_sheets_connector.py index d64aa89e7..bb344f721 100644 --- a/toucan_connectors/google_sheets/google_sheets_connector.py +++ b/toucan_connectors/google_sheets/google_sheets_connector.py @@ -35,6 +35,8 @@ class GoogleSheetsDataSource(ToucanDataSource): True, title='Dates as floats', description='Render Date as Floats or String from the sheet' ) + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheetsDataSource']) -> None: diff --git a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py index 6c39e9fbf..bb83122aa 100644 --- a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py +++ b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py @@ -67,6 +67,8 @@ class GoogleSheets2DataSource(ToucanDataSource): parameters: dict = Field(None, description='Additional URL parameters') parse_dates: List[str] = Field([], title='Dates column', description='Columns to parse as date') + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheets2DataSource']) -> None: @@ -98,7 +100,7 @@ class GoogleSheets2Connector(ToucanConnector): _auth_flow = 'oauth2' _oauth_trigger = 'instance' oauth2_version = Field('1', **{'ui.hidden': True}) - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None # TODO: turn into a class property _baseroute = 'https://sheets.googleapis.com/v4/spreadsheets/' diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index 3606561d9..67567fa3f 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -95,6 +95,8 @@ class HttpAPIDataSource(ToucanDataSource): filter: str = FilterSchema flatten_column: str = Field(None, description='Column containing nested rows') + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['HttpAPIDataSource']) -> None: diff --git a/toucan_connectors/hubspot/hubspot_connector.py b/toucan_connectors/hubspot/hubspot_connector.py index 9125ce0ca..7bd2d45a2 100644 --- a/toucan_connectors/hubspot/hubspot_connector.py +++ b/toucan_connectors/hubspot/hubspot_connector.py @@ -70,7 +70,7 @@ class HubspotDataSource(ToucanDataSource): class HubspotConnector(ToucanConnector): _auth_flow = 'oauth2' - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None _oauth_trigger = 'instance' oauth2_version = Field('1', **{'ui.hidden': True}) data_source_model: HubspotDataSource diff --git a/toucan_connectors/hubspot_private_app/hubspot_connector.py b/toucan_connectors/hubspot_private_app/hubspot_connector.py index 81ab3b955..73ae32617 100644 --- a/toucan_connectors/hubspot_private_app/hubspot_connector.py +++ b/toucan_connectors/hubspot_private_app/hubspot_connector.py @@ -4,7 +4,7 @@ import pandas as pd from hubspot import HubSpot # type:ignore[import] -from pydantic import BaseModel, Field, SecretStr +from pydantic import ConfigDict, BaseModel, Field, SecretStr from toucan_connectors.pagination import build_pagination_info from toucan_connectors.toucan_connector import DataSlice, ToucanConnector, ToucanDataSource @@ -32,15 +32,11 @@ class _HubSpotPaging(BaseModel): class _HubSpotResult(BaseModel): - created_at: datetime | None - updated_at: datetime | None + created_at: datetime | None = None + updated_at: datetime | None = None id_: str = Field(..., alias='id') properties: dict[str, Any] = Field(default_factory=dict) - - # For basic_api objects, properties are in a 'properties' object. For specialized APIs, such as - # owners, they're keys of the root object - class Config: - extra = 'allow' + model_config = ConfigDict(extra='allow') def to_dict(self) -> dict[str, Any]: dict_ = self.dict(by_alias=True) diff --git a/toucan_connectors/linkedinads/linkedinads_connector.py b/toucan_connectors/linkedinads/linkedinads_connector.py index 261c0445a..8f27b5244 100644 --- a/toucan_connectors/linkedinads/linkedinads_connector.py +++ b/toucan_connectors/linkedinads/linkedinads_connector.py @@ -73,6 +73,8 @@ class LinkedinadsDataSource(ToucanDataSource): description='See https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting for more information', ) + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['LinkedinadsDataSource']) -> None: @@ -96,7 +98,7 @@ class LinkedinadsConnector(ToucanConnector): _auth_flow = 'oauth2' auth_flow_id: Optional[ str - ] # This ID is generated & provided to the data provider during the oauth authentication process + ] = None # This ID is generated & provided to the data provider during the oauth authentication process _baseroute = 'https://api.linkedin.com/v2/adAnalyticsV2?q=' template: Template = Field( None, diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 036b01b4f..0798669ed 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -5,7 +5,7 @@ import pymongo from bson.son import SON from cached_property import cached_property -from pydantic import Field, SecretStr, create_model, validator +from pydantic import ConfigDict, Field, SecretStr, create_model, validator from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query from toucan_connectors.json_wrapper import JsonWrapper @@ -152,10 +152,10 @@ class MongoConnector(ToucanConnector, VersionableEngineConnector): username: Optional[str] = Field(None, description='Your login username') password: Optional[SecretStr] = Field(None, description='Your login password') ssl: Optional[bool] = Field(None, description='Create the connection to the server using SSL') + model_config = ConfigDict(ignored_types=(cached_property, _lru_cache_wrapper)) - class Config: - keep_untouched = (cached_property, _lru_cache_wrapper) - + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator('password') def password_must_have_a_user(cls, password, values): if password is not None and values['username'] is None: diff --git a/toucan_connectors/mssql/mssql_connector.py b/toucan_connectors/mssql/mssql_connector.py index d66d9ce22..1d06cf1d6 100644 --- a/toucan_connectors/mssql/mssql_connector.py +++ b/toucan_connectors/mssql/mssql_connector.py @@ -1,10 +1,11 @@ from contextlib import suppress import pyodbc -from pydantic import Field, SecretStr, constr, create_model +from pydantic import StringConstraints, Field, SecretStr, create_model from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from typing_extensions import Annotated class MSSQLDataSource(ToucanDataSource): @@ -15,13 +16,13 @@ class MSSQLDataSource(ToucanDataSource): description='The name of the database you want to query. ' "By default SQL Server selects the user's default database", ) - table: constr(min_length=1) = Field( + table: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='The name of the data table that you want to ' 'get (equivalent to "SELECT * FROM ' 'your_table")', ) - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' diff --git a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py index 6bc1e6b3d..e7935106a 100644 --- a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py +++ b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py @@ -2,10 +2,11 @@ from contextlib import suppress import pyodbc -from pydantic import Field, SecretStr, constr, create_model +from pydantic import StringConstraints, Field, SecretStr, create_model from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from typing_extensions import Annotated class MSSQLDataSource(ToucanDataSource): @@ -16,13 +17,13 @@ class MSSQLDataSource(ToucanDataSource): description='The name of the database you want to query. ' "By default SQL Server selects the user's default database", ) - table: constr(min_length=1) = Field( + table: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='The name of the data table that you want to ' 'get (equivalent to "SELECT * FROM ' 'your_table")', ) - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index aa9dea877..649972cfa 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -7,7 +7,7 @@ import pandas as pd import pymysql from cached_property import cached_property_with_ttl -from pydantic import Field, SecretStr, constr, create_model, validator +from pydantic import StringConstraints, ConfigDict, Field, SecretStr, create_model, validator from pymysql.constants import CR, ER from toucan_connectors.common import ConnectorStatus, pandas_read_sql @@ -21,6 +21,7 @@ strlist_to_enum, ) from toucan_connectors.utils.pem import sanitize_spaces_pem +from typing_extensions import Annotated def handle_date_0(df: pd.DataFrame) -> pd.DataFrame: @@ -49,7 +50,7 @@ class MySQLDataSource(ToucanDataSource): table: str = Field( None, **{'ui.hidden': True} ) # To avoid previous config migrations, won't be used - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' @@ -142,11 +143,12 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo description='SSL Mode to use to connect to the MySQL server. ' 'Equivalent of the --ssl-mode option of the MySQL client. Must be set in order to use SSL', ) + # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)) - class Config: - underscore_attrs_are_private = True - keep_untouched = (cached_property_with_ttl,) - + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator('ssl_key') @classmethod def ssl_key_validator(cls, ssl_key: str, values: dict) -> str: diff --git a/toucan_connectors/odbc/odbc_connector.py b/toucan_connectors/odbc/odbc_connector.py index ab587cf0b..a76c87dca 100644 --- a/toucan_connectors/odbc/odbc_connector.py +++ b/toucan_connectors/odbc/odbc_connector.py @@ -1,13 +1,14 @@ import pandas as pd import pyodbc -from pydantic import Field, constr +from pydantic import StringConstraints, Field from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from typing_extensions import Annotated class OdbcDataSource(ToucanDataSource): - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write your SQL query here', widget='sql' ) diff --git a/toucan_connectors/one_drive/one_drive_connector.py b/toucan_connectors/one_drive/one_drive_connector.py index 692cda3ab..b253a0249 100755 --- a/toucan_connectors/one_drive/one_drive_connector.py +++ b/toucan_connectors/one_drive/one_drive_connector.py @@ -45,7 +45,7 @@ class OneDriveDataSource(ToucanDataSource): description='Read one sheet or append multiple sheets', placeholder='Enter a sheet or a comma separated list of sheets', ) - range: Optional[str] + range: Optional[str] = None table: Optional[str] = Field( None, Title='Tables', @@ -77,7 +77,7 @@ class OneDriveConnector(ToucanConnector): _auth_flow = 'oauth2' _oauth_trigger = 'connector' oauth2_version = Field('1', **{'ui.hidden': True}) - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None authorization_url: str = Field(None, **{'ui.hidden': True}) token_url: str = Field(None, **{'ui.hidden': True}) diff --git a/toucan_connectors/oracle_sql/oracle_sql_connector.py b/toucan_connectors/oracle_sql/oracle_sql_connector.py index 5d891e81a..c5ba564e8 100644 --- a/toucan_connectors/oracle_sql/oracle_sql_connector.py +++ b/toucan_connectors/oracle_sql/oracle_sql_connector.py @@ -2,17 +2,18 @@ import cx_Oracle import pandas as pd -from pydantic import Field, SecretStr, constr, create_model +from pydantic import StringConstraints, Field, SecretStr, create_model from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from typing_extensions import Annotated class OracleSQLDataSource(ToucanDataSource): - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write your SQL query here', widget='sql' ) - table: constr(min_length=1) = Field( + table: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='The name of the data table that you want to ' 'get (equivalent to "SELECT * FROM "your_table")', diff --git a/toucan_connectors/pagination.py b/toucan_connectors/pagination.py index c91dd4ad9..b7cb464ba 100644 --- a/toucan_connectors/pagination.py +++ b/toucan_connectors/pagination.py @@ -10,7 +10,7 @@ class OffsetLimitInfo(BaseModel): """ offset: int - limit: int | None + limit: int | None = None class UnknownSizeDatasetPaginationInfo(BaseModel): @@ -48,8 +48,8 @@ class PaginationInfo(BaseModel): parameters: OffsetLimitInfo pagination_info: UnknownSizeDatasetPaginationInfo | KnownSizeDatasetPaginationInfo - next_page: OffsetLimitInfo | None - previous_page: OffsetLimitInfo | None + next_page: OffsetLimitInfo | None = None + previous_page: OffsetLimitInfo | None = None def build_pagination_info( diff --git a/toucan_connectors/peakina/peakina_connector.py b/toucan_connectors/peakina/peakina_connector.py index 1542fa155..44e4144ce 100644 --- a/toucan_connectors/peakina/peakina_connector.py +++ b/toucan_connectors/peakina/peakina_connector.py @@ -4,11 +4,11 @@ from peakina.datasource import DataSource from toucan_connectors.toucan_connector import ToucanConnector +from pydantic import ConfigDict class PeakinaDataSource(DataSource): - class Config: - extra = 'allow' + model_config = ConfigDict(extra='allow') def __init__(self, **data: Any) -> None: super().__init__(**data) diff --git a/toucan_connectors/postgres/postgresql_connector.py b/toucan_connectors/postgres/postgresql_connector.py index 1866eddda..c17d22f2f 100644 --- a/toucan_connectors/postgres/postgresql_connector.py +++ b/toucan_connectors/postgres/postgresql_connector.py @@ -2,7 +2,7 @@ from typing import Dict, List, Optional import psycopg2 as pgsql -from pydantic import Field, SecretStr, constr, create_model +from pydantic import StringConstraints, Field, SecretStr, create_model from toucan_connectors.common import ConnectorStatus, pandas_read_sql from toucan_connectors.postgres.utils import build_database_model_extraction_query, types @@ -15,6 +15,7 @@ VersionableEngineConnector, strlist_to_enum, ) +from typing_extensions import Annotated DEFAULT_DATABASE = 'postgres' @@ -23,7 +24,7 @@ class PostgresDataSource(ToucanDataSource): database: str = Field( DEFAULT_DATABASE, description='The name of the database you want to query' ) - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' @@ -35,7 +36,7 @@ class PostgresDataSource(ToucanDataSource): description='An object describing a simple select query' 'This field is used internally', **{'ui.hidden': True}, ) - table: constr(min_length=1) = Field( + table: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='The name of the data table that you want to ' 'get (equivalent to "SELECT * FROM ' diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index 0914e28a3..109122ce9 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -7,7 +7,7 @@ import pandas as pd import redshift_connector -from pydantic import Field, SecretStr, create_model, root_validator, validator +from pydantic import field_validator, StringConstraints, Field, SecretStr, create_model, root_validator from pydantic.types import constr from toucan_connectors.common import ConnectorStatus @@ -22,6 +22,7 @@ ToucanDataSource, strlist_to_enum, ) +from typing_extensions import Annotated TABLE_QUERY = """SELECT DISTINCT tablename FROM pg_table_def WHERE schemaname = 'public';""" @@ -69,7 +70,7 @@ class RedshiftDataSource(ToucanDataSource): database: str = Field( DEFAULT_DATABASE, description='The name of the database you want to query' ) - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' @@ -144,6 +145,8 @@ class RedshiftConnector(ToucanConnector, DiscoverableConnector): profile: str | None = Field(None, description='AWS profile') region: str | None = Field(None, description='The region in which there is your aws account.') + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: underscore_attrs_are_private = True keep_untouched = (cached_property,) @@ -156,7 +159,8 @@ def schema_extra(schema: dict[str, Any]) -> None: def available_dbs(self) -> list[str]: return self._list_db_names() - @validator('host') + @field_validator('host') + @classmethod def host_validator(cls, v): return re.sub(r'^https?://', '', v) diff --git a/toucan_connectors/salesforce/salesforce_connector.py b/toucan_connectors/salesforce/salesforce_connector.py index afe8c5766..2c78aa8b8 100644 --- a/toucan_connectors/salesforce/salesforce_connector.py +++ b/toucan_connectors/salesforce/salesforce_connector.py @@ -51,7 +51,7 @@ class SalesforceDataSource(ToucanDataSource): class SalesforceConnector(ToucanConnector): _auth_flow = 'oauth2' - auth_flow_id: Optional[str] + auth_flow_id: Optional[str] = None data_source_model: SalesforceDataSource instance_url: str = Field( None, diff --git a/toucan_connectors/sap_hana/sap_hana_connector.py b/toucan_connectors/sap_hana/sap_hana_connector.py index 0cfeb4c25..b447daf28 100644 --- a/toucan_connectors/sap_hana/sap_hana_connector.py +++ b/toucan_connectors/sap_hana/sap_hana_connector.py @@ -1,12 +1,13 @@ import pyhdb -from pydantic import Field, SecretStr, constr +from pydantic import StringConstraints, Field, SecretStr from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from typing_extensions import Annotated class SapHanaDataSource(ToucanDataSource): - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write your SQL query here', widget='sql' ) diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 0b971719f..48faa9749 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -151,6 +151,8 @@ class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConne ) category: Category = Field(Category.SNOWFLAKE, title='category', ui={'checkbox': False}) + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: dict[str, Any], model: Type['SnowflakeConnector']) -> None: diff --git a/toucan_connectors/snowflake_common.py b/toucan_connectors/snowflake_common.py index 5792f3cbe..69a406040 100644 --- a/toucan_connectors/snowflake_common.py +++ b/toucan_connectors/snowflake_common.py @@ -5,13 +5,14 @@ from typing import Any, Dict, List, Optional import pandas as pd -from pydantic import Field, constr +from pydantic import StringConstraints, Field from snowflake.connector import DictCursor, SnowflakeConnection from toucan_connectors.pagination import build_pagination_info from toucan_connectors.query_manager import QueryManager from toucan_connectors.sql_query_helper import SqlQueryHelper from toucan_connectors.toucan_connector import DataSlice, DataStats, QueryMetadata, ToucanDataSource +from typing_extensions import Annotated type_code_mapping = { 0: 'float', @@ -43,7 +44,7 @@ class SfDataSource(ToucanDataSource): database: str = Field(..., description='The name of the database you want to query') warehouse: str = Field(None, description='The name of the warehouse you want to query') - query: constr(min_length=1) = Field( + query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write your SQL query here', widget='sql' ) diff --git a/toucan_connectors/soap/soap_connector.py b/toucan_connectors/soap/soap_connector.py index 10953c439..e9e63f183 100644 --- a/toucan_connectors/soap/soap_connector.py +++ b/toucan_connectors/soap/soap_connector.py @@ -20,6 +20,8 @@ class SoapDataSource(ToucanDataSource): ) flatten_column: str = Field(None, description='Column containing nested rows') + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: @staticmethod def schema_extra(schema: Dict[str, Any], model: Type['SoapDataSource']) -> None: diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index 2d1c30fa6..84d0e427f 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -8,11 +8,11 @@ from abc import ABC, ABCMeta, abstractmethod from enum import Enum from functools import reduce, wraps -from typing import Any, Generic, Iterable, NamedTuple, Type, TypeVar, Union +from typing import Annotated, Any, Generic, Iterable, NamedTuple, Type, TypeVar, Union import pandas as pd import tenacity as tny -from pydantic import BaseModel, Field, SecretBytes, SecretStr +from pydantic import ConfigDict, BaseModel, Field, PlainSerializer, SecretBytes, SecretStr from toucan_connectors.common import ( ConnectorStatus, @@ -96,10 +96,7 @@ class ToucanDataSource(BaseModel, Generic[C]): title="Slow Queries' Cache Expiration Time", description='In seconds. Will override the 5min instance default and/or the connector value', ) - - class Config: - extra = 'forbid' - validate_assignment = True + model_config = ConfigDict(extra='forbid', validate_assignment=True) @classmethod def get_form(cls, connector: C, current_config): @@ -284,7 +281,7 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): name: str = Field(...) retry_policy: RetryPolicy | None = RetryPolicy() _retry_on: Iterable[Type[BaseException]] = () - type: str | None + type: str | None = None secrets_storage_version: str = Field('1', **_UI_HIDDEN) # type:ignore[pydantic-field] # Default ttl for all connector's queries (overridable at the data_source level) @@ -297,14 +294,16 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): # Used to defined the connection identifier: str = Field(None, **_UI_HIDDEN) # type:ignore[pydantic-field] - - class Config: - extra = 'forbid' - validate_assignment = True - json_encoders = { + # TODO[pydantic]: The following keys were removed: `json_encoders`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict( + extra='forbid', + validate_assignment=True, + json_encoders={ SecretStr: lambda v: v.get_secret_value(), SecretBytes: lambda v: v.get_secret_value(), - } + }, + ) @classmethod def __init_subclass__(cls): From 51bb0648b0c764d25ccadd5506d8efe80e290c1b Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 10:36:37 +0200 Subject: [PATCH 03/27] refactor: replace json_serializers Signed-off-by: Luka Peschke --- toucan_connectors/anaplan/anaplan_connector.py | 18 +++++++++++++----- .../awsathena/awsathena_connector.py | 11 +++++++---- .../azure_mssql/azure_mssql_connector.py | 8 ++++---- .../clickhouse/clickhouse_connector.py | 11 ++++++++--- .../databricks/databricks_connector.py | 6 +++--- .../elasticsearch/elasticsearch_connector.py | 6 +++--- .../google_cloud_mysql_connector.py | 8 ++++---- .../google_sheets/google_sheets_connector.py | 11 ++++++++--- .../hubspot_private_app/hubspot_connector.py | 13 ++++++++++--- .../micro_strategy/micro_strategy_connector.py | 8 ++++---- toucan_connectors/mongo/mongo_connector.py | 5 +++-- toucan_connectors/mssql/mssql_connector.py | 11 ++++++++--- .../mssql_TLSv1_0/mssql_connector.py | 11 ++++++++--- toucan_connectors/mysql/mysql_connector.py | 15 +++++++++------ .../net_explorer/net_explorer_connector.py | 6 +++--- .../oauth2_connector/oauth2connector.py | 5 +++-- .../one_drive/one_drive_connector.py | 6 +++--- .../oracle_sql/oracle_sql_connector.py | 11 ++++++++--- .../postgres/postgresql_connector.py | 5 +++-- .../redshift/redshift_database_connector.py | 7 ++++--- .../sap_hana/sap_hana_connector.py | 10 ++++++---- .../snowflake/snowflake_connector.py | 5 +++-- .../snowflake_oauth2_connector.py | 5 +++-- toucan_connectors/toucan_connector.py | 16 ++++++---------- 24 files changed, 134 insertions(+), 84 deletions(-) diff --git a/toucan_connectors/anaplan/anaplan_connector.py b/toucan_connectors/anaplan/anaplan_connector.py index 11f5d14f4..7e66a5f85 100644 --- a/toucan_connectors/anaplan/anaplan_connector.py +++ b/toucan_connectors/anaplan/anaplan_connector.py @@ -5,10 +5,14 @@ import pydantic import requests from pydantic import StringConstraints, Field, create_model -from pydantic.types import SecretStr from toucan_connectors.common import ConnectorStatus -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from toucan_connectors.toucan_connector import ( + ToucanConnector, + ToucanDataSource, + strlist_to_enum, + PlainJsonSecretStr, +) from typing_extensions import Annotated _ID_SEPARATOR = ' - ' @@ -23,8 +27,12 @@ def _format_name_and_id(obj: Dict[str, str]) -> str: class AnaplanDataSource(ToucanDataSource): - model_id: Annotated[str, StringConstraints(min_length=1)] = Field(..., description='The model you want to query') - view_id: Annotated[str, StringConstraints(min_length=1)] = Field(..., description='The view you want to query') + model_id: Annotated[str, StringConstraints(min_length=1)] = Field( + ..., description='The model you want to query' + ) + view_id: Annotated[str, StringConstraints(min_length=1)] = Field( + ..., description='The view you want to query' + ) workspace_id: str = Field(..., description='The ID of the workspace you want to query') @pydantic.validator('model_id', 'view_id', 'workspace_id') @@ -91,7 +99,7 @@ class AnaplanAuthError(AnaplanError): class AnaplanConnector(ToucanConnector): data_source_model: AnaplanDataSource username: str - password: Optional[SecretStr] = None + password: Optional[PlainJsonSecretStr] = None def _extract_json(self, resp: requests.Response) -> dict: if resp.status_code in (401, 403): diff --git a/toucan_connectors/awsathena/awsathena_connector.py b/toucan_connectors/awsathena/awsathena_connector.py index 8d00472fd..a813885f8 100644 --- a/toucan_connectors/awsathena/awsathena_connector.py +++ b/toucan_connectors/awsathena/awsathena_connector.py @@ -4,12 +4,13 @@ import boto3 import pandas as pd from cached_property import cached_property_with_ttl -from pydantic import StringConstraints, ConfigDict, Field, SecretStr, create_model +from pydantic import StringConstraints, ConfigDict, Field, create_model from toucan_connectors.common import ConnectorStatus, apply_query_parameters, sanitize_query from toucan_connectors.pagination import build_pagination_info from toucan_connectors.pandas_translator import PandasConditionTranslator from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, DataSlice, DataStats, DiscoverableConnector, @@ -76,17 +77,19 @@ class AwsathenaConnector(ToucanConnector, DiscoverableConnector): ..., description='Your S3 Output bucket (where query results are stored.)' ) aws_access_key_id: str = Field(..., description='Your AWS access key ID') - aws_secret_access_key: SecretStr = Field(None, description='Your AWS secret key') + aws_secret_access_key: PlainJsonSecretStr = Field(None, description='Your AWS secret key') region_name: str = Field(..., description='Your AWS region name') # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)) + model_config = ConfigDict( + underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,) + ) def get_session(self) -> boto3.Session: return boto3.Session( aws_access_key_id=self.aws_access_key_id, # This is required because this gets appended by boto3 - # internally, and a SecretStr can't be appended to an str + # internally, and a PlainJsonSecretStr can't be appended to an str aws_secret_access_key=self.aws_secret_access_key.get_secret_value(), region_name=self.region_name, ) diff --git a/toucan_connectors/azure_mssql/azure_mssql_connector.py b/toucan_connectors/azure_mssql/azure_mssql_connector.py index 56a94b8c2..57ed8482a 100644 --- a/toucan_connectors/azure_mssql/azure_mssql_connector.py +++ b/toucan_connectors/azure_mssql/azure_mssql_connector.py @@ -2,10 +2,10 @@ import pandas as pd import pyodbc -from pydantic import StringConstraints, Field, SecretStr +from pydantic import StringConstraints, Field from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr from typing_extensions import Annotated CLOUD_HOST = 'database.windows.net' @@ -32,7 +32,7 @@ class AzureMSSQLConnector(ToucanConnector): ) user: str = Field(..., description='Your login username') - password: SecretStr = Field('', description='Your login password') + password: PlainJsonSecretStr = Field('', description='Your login password') connect_timeout: int = Field( None, title='Connection timeout', @@ -45,7 +45,7 @@ def get_connection_params(self, *, database=None): user = f'{self.user}@{base_host}' if '@' not in self.user else self.user if not self.password: - self.password = SecretStr('') + self.password = PlainJsonSecretStr('') con_params = { 'driver': '{ODBC Driver 17 for SQL Server}', diff --git a/toucan_connectors/clickhouse/clickhouse_connector.py b/toucan_connectors/clickhouse/clickhouse_connector.py index eed1268a7..94ad99df5 100644 --- a/toucan_connectors/clickhouse/clickhouse_connector.py +++ b/toucan_connectors/clickhouse/clickhouse_connector.py @@ -2,10 +2,15 @@ from typing import Any, Dict, Type import clickhouse_driver -from pydantic import StringConstraints, Field, SecretStr, create_model +from pydantic import StringConstraints, Field, create_model from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from toucan_connectors.toucan_connector import ( + ToucanConnector, + ToucanDataSource, + strlist_to_enum, + PlainJsonSecretStr, +) from typing_extensions import Annotated @@ -93,7 +98,7 @@ class ClickhouseConnector(ToucanConnector): ) port: int = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field('', description='Your login password') + password: PlainJsonSecretStr = Field('', description='Your login password') ssl_connection: bool = Field(False, description='Create a SSL wrapped TCP connection') def get_connection_url(self, *, database='default'): diff --git a/toucan_connectors/databricks/databricks_connector.py b/toucan_connectors/databricks/databricks_connector.py index c0ad93333..7abb4ffea 100644 --- a/toucan_connectors/databricks/databricks_connector.py +++ b/toucan_connectors/databricks/databricks_connector.py @@ -4,11 +4,11 @@ import pandas as pd import pyodbc import requests -from pydantic import StringConstraints, Field, SecretStr +from pydantic import StringConstraints, Field from requests.auth import HTTPBasicAuth from toucan_connectors.common import ClusterStartException, ConnectorStatus, pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr from typing_extensions import Annotated logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class DatabricksConnector(ToucanConnector): http_path: str = Field( ..., description='Databricks compute resources URL', placeholder='sql/protocolv1/o/xxx/yyy' ) - pwd: SecretStr = Field( + pwd: PlainJsonSecretStr = Field( None, description='Your personal access token', placeholder='dapixxxxxxxxxxx' ) ansi: bool = False diff --git a/toucan_connectors/elasticsearch/elasticsearch_connector.py b/toucan_connectors/elasticsearch/elasticsearch_connector.py index 288b9521b..21db5719a 100644 --- a/toucan_connectors/elasticsearch/elasticsearch_connector.py +++ b/toucan_connectors/elasticsearch/elasticsearch_connector.py @@ -6,10 +6,10 @@ import pandas as pd from elasticsearch import Elasticsearch from pandas.io.json import json_normalize -from pydantic import BaseModel, Field, SecretStr +from pydantic import BaseModel, Field from toucan_connectors.common import nosql_apply_parameters_to_query -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr def _is_branch_list(val): @@ -102,7 +102,7 @@ class ElasticsearchHost(BaseModel): port: int = None scheme: str = None username: str = None - password: SecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr = Field(None, description='Your login password') headers: dict = None diff --git a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py index ffb5c65af..6fb8dcb58 100644 --- a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py +++ b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py @@ -1,9 +1,9 @@ import pandas as pd import pymysql -from pydantic import StringConstraints, Field, SecretStr +from pydantic import StringConstraints, Field from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr from typing_extensions import Annotated @@ -29,7 +29,7 @@ class GoogleCloudMySQLConnector(ToucanConnector): port: int = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field('', description='Your login password') + password: PlainJsonSecretStr = Field('', description='Your login password') charset: str = Field( 'utf8mb4', title='Charset', @@ -50,7 +50,7 @@ def get_connection_params(self, *, database=None): 'user': self.user, 'password': self.password.get_secret_value() if self.password - else SecretStr('').get_secret_value(), + else PlainJsonSecretStr('').get_secret_value(), 'port': self.port, 'database': database, 'charset': self.charset, diff --git a/toucan_connectors/google_sheets/google_sheets_connector.py b/toucan_connectors/google_sheets/google_sheets_connector.py index bb344f721..d76248e14 100644 --- a/toucan_connectors/google_sheets/google_sheets_connector.py +++ b/toucan_connectors/google_sheets/google_sheets_connector.py @@ -8,10 +8,15 @@ from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.errors import Error as GoogleApiClientError -from pydantic import Field, PrivateAttr, SecretStr, create_model +from pydantic import Field, PrivateAttr, create_model from toucan_connectors.common import ConnectorStatus -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from toucan_connectors.toucan_connector import ( + ToucanConnector, + ToucanDataSource, + strlist_to_enum, + PlainJsonSecretStr, +) class GoogleSheetsDataSource(ToucanDataSource): @@ -72,7 +77,7 @@ class GoogleSheetsConnector(ToucanConnector): _oauth_trigger = 'retrieve_token' _retrieve_token: Callable[[str, str], str] = PrivateAttr() - auth_id: SecretStr = None + auth_id: PlainJsonSecretStr = None def __init__(self, retrieve_token: Callable[[str, str], str], *args, **kwargs): super().__init__(**kwargs) diff --git a/toucan_connectors/hubspot_private_app/hubspot_connector.py b/toucan_connectors/hubspot_private_app/hubspot_connector.py index 73ae32617..80d906746 100644 --- a/toucan_connectors/hubspot_private_app/hubspot_connector.py +++ b/toucan_connectors/hubspot_private_app/hubspot_connector.py @@ -4,10 +4,15 @@ import pandas as pd from hubspot import HubSpot # type:ignore[import] -from pydantic import ConfigDict, BaseModel, Field, SecretStr +from pydantic import ConfigDict, BaseModel, Field from toucan_connectors.pagination import build_pagination_info -from toucan_connectors.toucan_connector import DataSlice, ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ( + DataSlice, + ToucanConnector, + ToucanDataSource, + PlainJsonSecretStr, +) class HubspotDataset(str, Enum): @@ -88,7 +93,9 @@ def _page_api_for(api: _Api, dataset: HubspotDataset) -> _PageApi: class HubspotConnector(ToucanConnector): data_source_model: HubspotDataSource - access_token: SecretStr = Field(..., description='An API key for the target private app') + access_token: PlainJsonSecretStr = Field( + ..., description='An API key for the target private app' + ) def _fetch_page( self, api: _PageApi, after: str | None = None, limit: int | None = None diff --git a/toucan_connectors/micro_strategy/micro_strategy_connector.py b/toucan_connectors/micro_strategy/micro_strategy_connector.py index 2451ef6c6..10eda4e1a 100644 --- a/toucan_connectors/micro_strategy/micro_strategy_connector.py +++ b/toucan_connectors/micro_strategy/micro_strategy_connector.py @@ -2,10 +2,10 @@ import pandas as pd from pandas.io.json import json_normalize -from pydantic import Field, HttpUrl, SecretStr +from pydantic import Field, HttpUrl from toucan_connectors.common import nosql_apply_parameters_to_query -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr from .client import Client from .data import ( @@ -71,7 +71,7 @@ class MicroStrategyConnector(ToucanConnector): examples=['https://demo.microstrategy.com/MicroStrategyLibrary2/api/'], ) username: str = Field(..., description='Your login username') - password: SecretStr = Field('', description='Your login password') + password: PlainJsonSecretStr = Field('', description='Your login password') project_id: str = Field( ..., title='projectID', @@ -98,7 +98,7 @@ def _retrieve_data(self, data_source: MicroStrategyDataSource) -> pd.DataFrame: if data_source.dataset == Dataset.search: return self._retrieve_metadata(data_source) if not self.password: - self.password = SecretStr('') + self.password = PlainJsonSecretStr('') client = Client( self.base_url, self.project_id, self.username, self.password.get_secret_value() ) diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 0798669ed..b18737825 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -5,13 +5,14 @@ import pymongo from bson.son import SON from cached_property import cached_property -from pydantic import ConfigDict, Field, SecretStr, create_model, validator +from pydantic import ConfigDict, Field, create_model, validator from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query from toucan_connectors.json_wrapper import JsonWrapper from toucan_connectors.mongo.mongo_translator import MongoConditionTranslator from toucan_connectors.pagination import build_pagination_info from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, DataSlice, ToucanConnector, ToucanDataSource, @@ -150,7 +151,7 @@ class MongoConnector(ToucanConnector, VersionableEngineConnector): ) port: Optional[int] = Field(None, description='The listening port of your database server') username: Optional[str] = Field(None, description='Your login username') - password: Optional[SecretStr] = Field(None, description='Your login password') + password: Optional[PlainJsonSecretStr] = Field(None, description='Your login password') ssl: Optional[bool] = Field(None, description='Create the connection to the server using SSL') model_config = ConfigDict(ignored_types=(cached_property, _lru_cache_wrapper)) diff --git a/toucan_connectors/mssql/mssql_connector.py b/toucan_connectors/mssql/mssql_connector.py index 1d06cf1d6..8453b8f47 100644 --- a/toucan_connectors/mssql/mssql_connector.py +++ b/toucan_connectors/mssql/mssql_connector.py @@ -1,10 +1,15 @@ from contextlib import suppress import pyodbc -from pydantic import StringConstraints, Field, SecretStr, create_model +from pydantic import StringConstraints, Field, create_model from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from toucan_connectors.toucan_connector import ( + ToucanConnector, + ToucanDataSource, + strlist_to_enum, + PlainJsonSecretStr, +) from typing_extensions import Annotated @@ -84,7 +89,7 @@ class MSSQLConnector(ToucanConnector): port: int = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr = Field(None, description='Your login password') connect_timeout: int = Field( None, title='Connection timeout', diff --git a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py index e7935106a..9a0db3192 100644 --- a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py +++ b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py @@ -2,10 +2,15 @@ from contextlib import suppress import pyodbc -from pydantic import StringConstraints, Field, SecretStr, create_model +from pydantic import StringConstraints, Field, create_model from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from toucan_connectors.toucan_connector import ( + ToucanConnector, + ToucanDataSource, + strlist_to_enum, + PlainJsonSecretStr, +) from typing_extensions import Annotated @@ -85,7 +90,7 @@ class MSSQLConnector(ToucanConnector): port: int = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr = Field(None, description='Your login password') connect_timeout: int = Field( None, title='Connection timeout', diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index 649972cfa..55d0685c1 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -7,7 +7,7 @@ import pandas as pd import pymysql from cached_property import cached_property_with_ttl -from pydantic import StringConstraints, ConfigDict, Field, SecretStr, create_model, validator +from pydantic import StringConstraints, ConfigDict, Field, create_model, validator from pymysql.constants import CR, ER from toucan_connectors.common import ConnectorStatus, pandas_read_sql @@ -19,6 +19,7 @@ UnavailableVersion, VersionableEngineConnector, strlist_to_enum, + PlainJsonSecretStr, ) from toucan_connectors.utils.pem import sanitize_spaces_pem from typing_extensions import Annotated @@ -109,7 +110,7 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo ) port: int = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr = Field(None, description='Your login password') charset: str = Field( 'utf8mb4', title='Charset', @@ -123,17 +124,17 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo 'for the server to respond. None by default', ) # SSL options - ssl_ca: SecretStr = Field( + ssl_ca: PlainJsonSecretStr = Field( None, description='The CA certificate content in PEM format to use to connect to the MySQL ' 'server. Equivalent of the --ssl-ca option of the MySQL client', ) - ssl_cert: SecretStr = Field( + ssl_cert: PlainJsonSecretStr = Field( None, description='The X509 certificate content in PEM format to use to connect to the MySQL ' 'server. Equivalent of the --ssl-cert option of the MySQL client', ) - ssl_key: SecretStr = Field( + ssl_key: PlainJsonSecretStr = Field( None, description='The X509 certificate key content in PEM format to use to connect to the MySQL ' 'server. Equivalent of the --ssl-key option of the MySQL client', @@ -145,7 +146,9 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo ) # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,)) + model_config = ConfigDict( + underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,) + ) # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. diff --git a/toucan_connectors/net_explorer/net_explorer_connector.py b/toucan_connectors/net_explorer/net_explorer_connector.py index dfb71dbfa..6d022636e 100755 --- a/toucan_connectors/net_explorer/net_explorer_connector.py +++ b/toucan_connectors/net_explorer/net_explorer_connector.py @@ -5,10 +5,10 @@ import pandas as pd import requests -from pydantic import Field, SecretStr +from pydantic import Field from toucan_connectors.common import ConnectorStatus -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr class NetExplorerDataSource(ToucanDataSource): @@ -24,7 +24,7 @@ class NetExplorerConnector(ToucanConnector): placeholder='exemple.netexplorer.pro', ) user: str - password: SecretStr + password: PlainJsonSecretStr def _retrieve_token(self): login_url = f'https://{self.instance_url}/api/auth' diff --git a/toucan_connectors/oauth2_connector/oauth2connector.py b/toucan_connectors/oauth2_connector/oauth2connector.py index 30eb9c7b9..8b79d2320 100644 --- a/toucan_connectors/oauth2_connector/oauth2connector.py +++ b/toucan_connectors/oauth2_connector/oauth2connector.py @@ -6,9 +6,10 @@ from authlib.common.security import generate_token from authlib.integrations.requests_client import OAuth2Session -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel from toucan_connectors.json_wrapper import JsonWrapper +from toucan_connectors.toucan_connector import PlainJsonSecretStr class SecretsKeeper(ABC): @@ -27,7 +28,7 @@ def load(self, key: str, **kwargs) -> Any: class OAuth2ConnectorConfig(BaseModel): client_id: str - client_secret: SecretStr + client_secret: PlainJsonSecretStr class OAuth2Connector: diff --git a/toucan_connectors/one_drive/one_drive_connector.py b/toucan_connectors/one_drive/one_drive_connector.py index b253a0249..de23ef662 100755 --- a/toucan_connectors/one_drive/one_drive_connector.py +++ b/toucan_connectors/one_drive/one_drive_connector.py @@ -4,14 +4,14 @@ import pandas as pd import requests -from pydantic import Field, PrivateAttr, SecretStr +from pydantic import Field, PrivateAttr from toucan_connectors.common import ConnectorStatus from toucan_connectors.oauth2_connector.oauth2connector import ( OAuth2Connector, OAuth2ConnectorConfig, ) -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr class NotFoundError(Exception): @@ -90,7 +90,7 @@ class OneDriveConnector(ToucanConnector): description='The client id of you Azure Active Directory integration', **{'ui.required': True}, ) - client_secret: SecretStr = Field( + client_secret: PlainJsonSecretStr = Field( '', title='Client Secret', description='The client secret of your Azure Active Directory integration', diff --git a/toucan_connectors/oracle_sql/oracle_sql_connector.py b/toucan_connectors/oracle_sql/oracle_sql_connector.py index c5ba564e8..5fbfcb729 100644 --- a/toucan_connectors/oracle_sql/oracle_sql_connector.py +++ b/toucan_connectors/oracle_sql/oracle_sql_connector.py @@ -2,10 +2,15 @@ import cx_Oracle import pandas as pd -from pydantic import StringConstraints, Field, SecretStr, create_model +from pydantic import StringConstraints, Field, create_model from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum +from toucan_connectors.toucan_connector import ( + ToucanConnector, + ToucanDataSource, + strlist_to_enum, + PlainJsonSecretStr, +) from typing_extensions import Annotated @@ -62,7 +67,7 @@ class OracleSQLConnector(ToucanConnector): examples=['localhost:80/service'], ) user: str = Field(None, description='Your login username') - password: SecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr = Field(None, description='Your login password') encoding: str = Field( None, title='Charset', description='If you need to specify a specific character encoding.' ) diff --git a/toucan_connectors/postgres/postgresql_connector.py b/toucan_connectors/postgres/postgresql_connector.py index c17d22f2f..4081c62ac 100644 --- a/toucan_connectors/postgres/postgresql_connector.py +++ b/toucan_connectors/postgres/postgresql_connector.py @@ -2,7 +2,7 @@ from typing import Dict, List, Optional import psycopg2 as pgsql -from pydantic import StringConstraints, Field, SecretStr, create_model +from pydantic import StringConstraints, Field, create_model from toucan_connectors.common import ConnectorStatus, pandas_read_sql from toucan_connectors.postgres.utils import build_database_model_extraction_query, types @@ -14,6 +14,7 @@ UnavailableVersion, VersionableEngineConnector, strlist_to_enum, + PlainJsonSecretStr, ) from typing_extensions import Annotated @@ -100,7 +101,7 @@ class PostgresConnector(ToucanConnector, DiscoverableConnector, VersionableEngin ) port: int = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr = Field(None, description='Your login password') default_database: str = Field(DEFAULT_DATABASE, description='Your default database') charset: str = Field(None, description='If you need to specify a specific character encoding.') diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index 109122ce9..08afc2fe1 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -7,7 +7,7 @@ import pandas as pd import redshift_connector -from pydantic import field_validator, StringConstraints, Field, SecretStr, create_model, root_validator +from pydantic import field_validator, StringConstraints, Field, create_model, root_validator from pydantic.types import constr from toucan_connectors.common import ConnectorStatus @@ -21,6 +21,7 @@ ToucanConnector, ToucanDataSource, strlist_to_enum, + PlainJsonSecretStr, ) from typing_extensions import Annotated @@ -114,7 +115,7 @@ class RedshiftConnector(ToucanConnector, DiscoverableConnector): user: str | None = Field( None, description='The username to use for authentication with the Amazon Redshift cluster' ) - password: SecretStr | None = Field( + password: PlainJsonSecretStr | None = Field( None, description='The password to use for authentication with the Amazon Redshift cluster' ) @@ -138,7 +139,7 @@ class RedshiftConnector(ToucanConnector, DiscoverableConnector): ) access_key_id: str | None = Field(None, description='The access key id of your aws account.') - secret_access_key: SecretStr | None = Field( + secret_access_key: PlainJsonSecretStr | None = Field( None, description='The secret access key of your aws account.' ) session_token: str | None = Field(None, description='Your session token') diff --git a/toucan_connectors/sap_hana/sap_hana_connector.py b/toucan_connectors/sap_hana/sap_hana_connector.py index b447daf28..91ca805a0 100644 --- a/toucan_connectors/sap_hana/sap_hana_connector.py +++ b/toucan_connectors/sap_hana/sap_hana_connector.py @@ -1,8 +1,8 @@ import pyhdb -from pydantic import StringConstraints, Field, SecretStr +from pydantic import StringConstraints, Field from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource +from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr from typing_extensions import Annotated @@ -27,14 +27,16 @@ class SapHanaConnector(ToucanConnector): port: int = Field(..., description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: SecretStr = Field('', description='Your login password') + password: PlainJsonSecretStr = Field('', description='Your login password') def _retrieve_data(self, data_source): connection = pyhdb.connect( self.host, self.port, self.user, - self.password.get_secret_value() if self.password else SecretStr('').get_secret_value(), + self.password.get_secret_value() + if self.password + else PlainJsonSecretStr('').get_secret_value(), ) df = pandas_read_sql(data_source.query, con=connection) diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 48faa9749..9532f42eb 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -9,7 +9,7 @@ import requests import snowflake from jinja2 import Template -from pydantic import Field, SecretStr, create_model +from pydantic import Field, create_model from snowflake import connector as sf_connector from snowflake.connector import SnowflakeConnection from snowflake.connector.cursor import DictCursor as SfDictCursor @@ -29,6 +29,7 @@ ToucanConnector, ToucanDataSource, strlist_to_enum, + PlainJsonSecretStr, ) logger = logging.getLogger(__name__) @@ -133,7 +134,7 @@ class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConne ) user: str = Field(..., description='Your login username') - password: SecretStr | None = Field(None, description='Your login password') + password: PlainJsonSecretStr | None = Field(None, description='Your login password') token_endpoint: str | None = Field(None, description='The token endpoint') token_endpoint_content_type: str = Field( 'application/json', diff --git a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py index 17a84cc00..3a8024e0a 100644 --- a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py +++ b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py @@ -7,7 +7,7 @@ import pandas as pd import snowflake -from pydantic import Field, PrivateAttr, SecretStr, create_model +from pydantic import Field, PrivateAttr, create_model from snowflake.connector import SnowflakeConnection from toucan_connectors.connection_manager import ConnectionManager @@ -28,6 +28,7 @@ DiscoverableConnector, ToucanConnector, strlist_to_enum, + PlainJsonSecretStr, ) logger = logging.getLogger(__name__) @@ -67,7 +68,7 @@ class SnowflakeoAuth2Connector(ToucanConnector): description='The client id of you Snowflake integration', **{'ui.required': True}, ) - client_secret: SecretStr = Field( + client_secret: PlainJsonSecretStr = Field( '', title='Client Secret', description='The client secret of your Snowflake integration', diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index 84d0e427f..e45b79782 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -258,6 +258,11 @@ def get_connector_secrets_form(cls) -> ConnectorSecretsForm | None: DS = TypeVar('DS', bound=ToucanDataSource) +PlainJsonSecretStr = Annotated[ + SecretStr, PlainSerializer(SecretStr.get_secret_value, return_type=str) +] + + class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): """Abstract base class for all toucan connectors. @@ -294,16 +299,7 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): # Used to defined the connection identifier: str = Field(None, **_UI_HIDDEN) # type:ignore[pydantic-field] - # TODO[pydantic]: The following keys were removed: `json_encoders`. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - model_config = ConfigDict( - extra='forbid', - validate_assignment=True, - json_encoders={ - SecretStr: lambda v: v.get_secret_value(), - SecretBytes: lambda v: v.get_secret_value(), - }, - ) + model_config = ConfigDict(extra='forbid', validate_assignment=True) @classmethod def __init_subclass__(cls): From 102eb89d493893946f31282f535763ca34e11cc4 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 10:38:03 +0200 Subject: [PATCH 04/27] fix: underscore_attrs_are_private Signed-off-by: Luka Peschke --- toucan_connectors/awsathena/awsathena_connector.py | 6 +----- .../google_big_query/google_big_query_connector.py | 4 +--- toucan_connectors/mysql/mysql_connector.py | 6 +----- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/toucan_connectors/awsathena/awsathena_connector.py b/toucan_connectors/awsathena/awsathena_connector.py index a813885f8..e0dcca21b 100644 --- a/toucan_connectors/awsathena/awsathena_connector.py +++ b/toucan_connectors/awsathena/awsathena_connector.py @@ -79,11 +79,7 @@ class AwsathenaConnector(ToucanConnector, DiscoverableConnector): aws_access_key_id: str = Field(..., description='Your AWS access key ID') aws_secret_access_key: PlainJsonSecretStr = Field(None, description='Your AWS secret key') region_name: str = Field(..., description='Your AWS region name') - # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - model_config = ConfigDict( - underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,) - ) + model_config = ConfigDict(ignored_types=(cached_property_with_ttl,)) def get_session(self) -> boto3.Session: return boto3.Session( diff --git a/toucan_connectors/google_big_query/google_big_query_connector.py b/toucan_connectors/google_big_query/google_big_query_connector.py index 55f74c77c..ed2ab3c5a 100644 --- a/toucan_connectors/google_big_query/google_big_query_connector.py +++ b/toucan_connectors/google_big_query/google_big_query_connector.py @@ -103,9 +103,7 @@ class GoogleBigQueryConnector(ToucanConnector, DiscoverableConnector): 'the Google APIs. For more information, see this ' 'documentation', ) - # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - model_config = ConfigDict(underscore_attrs_are_private=True, ignored_types=(cached_property,)) + model_config = ConfigDict(ignored_types=(cached_property,)) @staticmethod def _get_google_credentials(credentials: GoogleCredentials, scopes: List[str]) -> Credentials: diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index 55d0685c1..92ad2a104 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -144,11 +144,7 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo description='SSL Mode to use to connect to the MySQL server. ' 'Equivalent of the --ssl-mode option of the MySQL client. Must be set in order to use SSL', ) - # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - model_config = ConfigDict( - underscore_attrs_are_private=True, ignored_types=(cached_property_with_ttl,) - ) + model_config = ConfigDict(ignored_types=(cached_property_with_ttl,)) # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. From 78e26f325e306f7d884b6b8b6e6648498e8715e4 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 11:29:22 +0200 Subject: [PATCH 05/27] refactor: replace schema_extra Signed-off-by: Luka Peschke --- .../clickhouse/clickhouse_connector.py | 39 +++++++----- .../google_adwords_connector.py | 45 +++++++++----- .../google_sheets/google_sheets_connector.py | 29 ++++++--- .../google_sheets_2_connector.py | 29 ++++++--- .../http_api/http_api_connector.py | 43 ++++++++----- .../linkedinads/linkedinads_connector.py | 43 ++++++++----- .../redshift/redshift_database_connector.py | 35 ++++++++--- .../snowflake/snowflake_connector.py | 62 ++++++++++++------- toucan_connectors/soap/soap_connector.py | 29 ++++++--- 9 files changed, 233 insertions(+), 121 deletions(-) diff --git a/toucan_connectors/clickhouse/clickhouse_connector.py b/toucan_connectors/clickhouse/clickhouse_connector.py index 94ad99df5..01122ae4c 100644 --- a/toucan_connectors/clickhouse/clickhouse_connector.py +++ b/toucan_connectors/clickhouse/clickhouse_connector.py @@ -3,6 +3,7 @@ import clickhouse_driver from pydantic import StringConstraints, Field, create_model +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ( @@ -30,20 +31,30 @@ class ClickhouseDataSource(ToucanDataSource): 'your_table")', ) - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['ClickhouseDataSource']) -> None: - keys = schema['properties'].keys() - prio_keys = [ - 'database', - 'table', - 'query', - 'parameters', - ] - new_keys = prio_keys + [k for k in keys if k not in prio_keys] - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + prio_keys = [ + 'database', + 'table', + 'query', + 'parameters', + ] + new_keys = prio_keys + [k for k in keys if k not in prio_keys] + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema def __init__(self, **data): super().__init__(**data) diff --git a/toucan_connectors/google_adwords/google_adwords_connector.py b/toucan_connectors/google_adwords/google_adwords_connector.py index 96388bc75..1d7132ba4 100644 --- a/toucan_connectors/google_adwords/google_adwords_connector.py +++ b/toucan_connectors/google_adwords/google_adwords_connector.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Optional, Type import pandas as pd +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode import requests from googleads import AdWordsClient, adwords, oauth2 from pydantic import Field, PrivateAttr @@ -59,23 +60,33 @@ class GoogleAdwordsDataSource(ToucanDataSource): description='Max number of rows to extract, for service extraction only', ) - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['GoogleAdwordsDataSource']) -> None: - keys = schema['properties'].keys() - prio_keys = [ - 'service', - 'columns', - 'from_clause', - 'parameters', - 'during', - 'orderby', - 'limit', - ] - new_keys = prio_keys + [k for k in keys if k not in prio_keys] - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + prio_keys = [ + 'service', + 'columns', + 'from_clause', + 'parameters', + 'during', + 'orderby', + 'limit', + ] + new_keys = prio_keys + [k for k in keys if k not in prio_keys] + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema class GoogleAdwordsConnector(ToucanConnector): diff --git a/toucan_connectors/google_sheets/google_sheets_connector.py b/toucan_connectors/google_sheets/google_sheets_connector.py index d76248e14..b37111ef9 100644 --- a/toucan_connectors/google_sheets/google_sheets_connector.py +++ b/toucan_connectors/google_sheets/google_sheets_connector.py @@ -9,6 +9,7 @@ from googleapiclient.discovery import build from googleapiclient.errors import Error as GoogleApiClientError from pydantic import Field, PrivateAttr, create_model +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from toucan_connectors.common import ConnectorStatus from toucan_connectors.toucan_connector import ( @@ -40,15 +41,25 @@ class GoogleSheetsDataSource(ToucanDataSource): True, title='Dates as floats', description='Render Date as Floats or String from the sheet' ) - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheetsDataSource']) -> None: - keys = schema['properties'].keys() - prio_keys = ['domain', 'spreadsheet_id', 'sheet'] - new_keys = prio_keys + [k for k in keys if k not in prio_keys] - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + prio_keys = ['domain', 'spreadsheet_id', 'sheet'] + new_keys = prio_keys + [k for k in keys if k not in prio_keys] + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema @classmethod def get_form(cls, connector: 'GoogleSheetsConnector', current_config, **kwargs): diff --git a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py index bb83122aa..4b9db665a 100644 --- a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py +++ b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py @@ -10,6 +10,7 @@ import pandas as pd from aiohttp import ClientSession from pydantic import Field, PrivateAttr, create_model +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from toucan_connectors.common import ConnectorStatus, HttpError, fetch, get_loop from toucan_connectors.oauth2_connector.oauth2connector import ( @@ -67,15 +68,25 @@ class GoogleSheets2DataSource(ToucanDataSource): parameters: dict = Field(None, description='Additional URL parameters') parse_dates: List[str] = Field([], title='Dates column', description='Columns to parse as date') - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['GoogleSheets2DataSource']) -> None: - keys = schema['properties'].keys() - prio_keys = ['domain', 'spreadsheet_id', 'sheet'] - new_keys = prio_keys + [k for k in keys if k not in prio_keys] - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + prio_keys = ['domain', 'spreadsheet_id', 'sheet'] + new_keys = prio_keys + [k for k in keys if k not in prio_keys] + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema @classmethod def get_form(cls, connector: 'GoogleSheets2Connector', current_config, **kwargs): diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index 67567fa3f..26690baf6 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -5,6 +5,7 @@ import pandas as pd from pydantic import AnyHttpUrl, BaseModel, Field, FilePath +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from requests import Session from toucan_data_sdk.utils.postprocess.json_to_table import json_to_table from xmltodict import parse @@ -95,22 +96,32 @@ class HttpAPIDataSource(ToucanDataSource): filter: str = FilterSchema flatten_column: str = Field(None, description='Column containing nested rows') - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['HttpAPIDataSource']) -> None: - keys = schema['properties'].keys() - last_keys = [ - 'proxies', - 'flatten_column', - 'data', - 'xpath', - 'filter', - 'validation', - ] - new_keys = [k for k in keys if k not in last_keys] + last_keys - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + last_keys = [ + 'proxies', + 'flatten_column', + 'data', + 'xpath', + 'filter', + 'validation', + ] + new_keys = [k for k in keys if k not in last_keys] + last_keys + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema class HttpAPIConnector(ToucanConnector): diff --git a/toucan_connectors/linkedinads/linkedinads_connector.py b/toucan_connectors/linkedinads/linkedinads_connector.py index 8f27b5244..282ad1478 100644 --- a/toucan_connectors/linkedinads/linkedinads_connector.py +++ b/toucan_connectors/linkedinads/linkedinads_connector.py @@ -7,6 +7,7 @@ import dateutil.parser import pandas as pd +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode import requests from pydantic import Field, PrivateAttr from toucan_data_sdk.utils.postprocess.json_to_table import json_to_table @@ -73,22 +74,32 @@ class LinkedinadsDataSource(ToucanDataSource): description='See https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting for more information', ) - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['LinkedinadsDataSource']) -> None: - keys = schema['properties'].keys() - prio_keys = [ - 'finder_methods', - 'start_date', - 'end_date', - 'time_granularity', - 'flatten_column', - 'parameters', - ] - new_keys = prio_keys + [k for k in keys if k not in prio_keys] - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + prio_keys = [ + 'finder_methods', + 'start_date', + 'end_date', + 'time_granularity', + 'flatten_column', + 'parameters', + ] + new_keys = prio_keys + [k for k in keys if k not in prio_keys] + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema class LinkedinadsConnector(ToucanConnector): diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index 08afc2fe1..d6f47a54c 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -6,8 +6,16 @@ from typing import Any import pandas as pd +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode import redshift_connector -from pydantic import field_validator, StringConstraints, Field, create_model, root_validator +from pydantic import ( + ConfigDict, + field_validator, + StringConstraints, + Field, + create_model, + root_validator, +) from pydantic.types import constr from toucan_connectors.common import ConnectorStatus @@ -146,15 +154,24 @@ class RedshiftConnector(ToucanConnector, DiscoverableConnector): profile: str | None = Field(None, description='AWS profile') region: str | None = Field(None, description='The region in which there is your aws account.') - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - underscore_attrs_are_private = True - keep_untouched = (cached_property,) + model_config = ConfigDict(ignored_types=(cached_property,)) - @staticmethod - def schema_extra(schema: dict[str, Any]) -> None: - schema['properties'] = {k: schema['properties'][k] for k in ORDERED_KEYS} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + schema['properties'] = {k: schema['properties'][k] for k in ORDERED_KEYS} + return schema @cached_property def available_dbs(self) -> list[str]: diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 9532f42eb..94efa5fe6 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -6,6 +6,12 @@ import jwt import pandas as pd +from pydantic.json_schema import ( + DEFAULT_REF_TEMPLATE, + GenerateJsonSchema, + JsonSchemaMode, + model_json_schema, +) import requests import snowflake from jinja2 import Template @@ -105,6 +111,24 @@ def _snowflake_connection( conn.close() +_SCHEMA_ORDERED_KEYS = ( + 'type', + 'name', + 'account', + 'authentication_method', + 'user', + 'password', + 'token_endpoint', + 'token_endpoint_content_type', + 'role', + 'default_warehouse', + 'retry_policy', + 'secrets_storage_version', + 'sso_credentials_keeper', + 'user_tokens_keeper', +) + + class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConnector): """ Import data from Snowflake data warehouse. @@ -152,28 +176,22 @@ class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConne ) category: Category = Field(Category.SNOWFLAKE, title='category', ui={'checkbox': False}) - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: dict[str, Any], model: Type['SnowflakeConnector']) -> None: - ordered_keys = [ - 'type', - 'name', - 'account', - 'authentication_method', - 'user', - 'password', - 'token_endpoint', - 'token_endpoint_content_type', - 'role', - 'default_warehouse', - 'retry_policy', - 'secrets_storage_version', - 'sso_credentials_keeper', - 'user_tokens_keeper', - ] - schema['properties'] = {k: schema['properties'][k] for k in ordered_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + schema['properties'] = {k: schema['properties'][k] for k in _SCHEMA_ORDERED_KEYS} + return schema @property def access_token(self) -> str | None: diff --git a/toucan_connectors/soap/soap_connector.py b/toucan_connectors/soap/soap_connector.py index e9e63f183..fbfa758f2 100644 --- a/toucan_connectors/soap/soap_connector.py +++ b/toucan_connectors/soap/soap_connector.py @@ -2,6 +2,7 @@ import pandas as pd from pydantic import Field, create_model +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from requests import Session from toucan_data_sdk.utils.postprocess.json_to_table import json_to_table from zeep import Client @@ -20,15 +21,25 @@ class SoapDataSource(ToucanDataSource): ) flatten_column: str = Field(None, description='Column containing nested rows') - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type['SoapDataSource']) -> None: - keys = schema['properties'].keys() - prio_keys = ['domain', 'method', 'parameters', 'flatten_column'] - new_keys = prio_keys + [k for k in keys if k not in prio_keys] - schema['properties'] = {k: schema['properties'][k] for k in new_keys} + @classmethod + def model_json_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = 'validation', + ) -> dict[str, Any]: + schema = super().model_json_schema( + by_alias=by_alias, + ref_template=ref_template, + schema_generator=schema_generator, + mode=mode, + ) + keys = schema['properties'].keys() + prio_keys = ['domain', 'method', 'parameters', 'flatten_column'] + new_keys = prio_keys + [k for k in keys if k not in prio_keys] + schema['properties'] = {k: schema['properties'][k] for k in new_keys} + return schema @classmethod def _get_methods_docs(cls, client): From a0915c0445dfa701493acd8b48bdc317632c9622 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 11:37:17 +0200 Subject: [PATCH 06/27] refactor validators with values Signed-off-by: Luka Peschke --- toucan_connectors/mongo/mongo_connector.py | 12 +++++------- toucan_connectors/mysql/mysql_connector.py | 16 ++++++---------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index b18737825..6b33c4ebf 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -5,7 +5,7 @@ import pymongo from bson.son import SON from cached_property import cached_property -from pydantic import ConfigDict, Field, create_model, validator +from pydantic import ConfigDict, Field, create_model, model_validator, validator from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query from toucan_connectors.json_wrapper import JsonWrapper @@ -155,13 +155,11 @@ class MongoConnector(ToucanConnector, VersionableEngineConnector): ssl: Optional[bool] = Field(None, description='Create the connection to the server using SSL') model_config = ConfigDict(ignored_types=(cached_property, _lru_cache_wrapper)) - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator('password') - def password_must_have_a_user(cls, password, values): - if password is not None and values['username'] is None: + @model_validator(mode='after') + def password_must_have_a_user(self) -> 'MongoConnector': + if self.password is not None and self.username is None: raise ValueError('username must be set') - return password + return self def __hash__(self): return hash(id(self)) + hash(JsonWrapper.dumps(self._get_mongo_client_kwargs())) diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index 92ad2a104..c85785b89 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -7,7 +7,7 @@ import pandas as pd import pymysql from cached_property import cached_property_with_ttl -from pydantic import StringConstraints, ConfigDict, Field, create_model, validator +from pydantic import StringConstraints, ConfigDict, Field, create_model, model_validator, validator from pymysql.constants import CR, ER from toucan_connectors.common import ConnectorStatus, pandas_read_sql @@ -146,19 +146,15 @@ class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineCo ) model_config = ConfigDict(ignored_types=(cached_property_with_ttl,)) - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator('ssl_key') - @classmethod - def ssl_key_validator(cls, ssl_key: str, values: dict) -> str: - ssl_cert = values.get('ssl_cert', None) + @model_validator(mode='after') + def ssl_key_validator(self) -> 'MySQLConnector': # if one is present, the other one should be specified - if ssl_cert is not None and ssl_key is None: + if self.ssl_cert is not None and self.ssl_key is None: raise ValueError('SSL option "ssl_key" should be specified if "ssl_cert" is provided !') - elif ssl_key is not None and ssl_cert is None: + elif self.ssl_key is not None and self.ssl_cert is None: raise ValueError('SSL option "ssl_cert" should be specified if "ssl_key" is provided !') - return ssl_key + return self def _sanitize_ssl_params(self) -> dict[str, Any]: params = {} From 8143eda09023c2d1d0d323d886a37d33ae13f8f7 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 11:41:32 +0200 Subject: [PATCH 07/27] chore: lint Signed-off-by: Luka Peschke --- toucan_connectors/anaplan/anaplan_connector.py | 6 +++--- toucan_connectors/awsathena/awsathena_connector.py | 6 +++--- .../azure_mssql/azure_mssql_connector.py | 6 +++--- .../clickhouse/clickhouse_connector.py | 8 ++++---- .../databricks/databricks_connector.py | 6 +++--- .../elasticsearch/elasticsearch_connector.py | 2 +- .../google_adwords/google_adwords_connector.py | 4 ++-- .../google_analytics/google_analytics_connector.py | 2 +- .../google_cloud_mysql_connector.py | 6 +++--- toucan_connectors/google_credentials.py | 2 +- .../google_sheets/google_sheets_connector.py | 4 ++-- .../google_sheets_2/google_sheets_2_connector.py | 2 +- toucan_connectors/http_api/http_api_connector.py | 2 +- .../hubspot_private_app/hubspot_connector.py | 4 ++-- .../linkedinads/linkedinads_connector.py | 4 ++-- .../micro_strategy/micro_strategy_connector.py | 2 +- toucan_connectors/mongo/mongo_connector.py | 4 ++-- toucan_connectors/mssql/mssql_connector.py | 6 +++--- toucan_connectors/mssql_TLSv1_0/mssql_connector.py | 6 +++--- toucan_connectors/mysql/mysql_connector.py | 6 +++--- .../net_explorer/net_explorer_connector.py | 2 +- toucan_connectors/odbc/odbc_connector.py | 4 ++-- toucan_connectors/one_drive/one_drive_connector.py | 2 +- .../oracle_sql/oracle_sql_connector.py | 6 +++--- toucan_connectors/peakina/peakina_connector.py | 2 +- toucan_connectors/postgres/postgresql_connector.py | 6 +++--- .../redshift/redshift_database_connector.py | 11 +++++------ toucan_connectors/sap_hana/sap_hana_connector.py | 6 +++--- toucan_connectors/snowflake/snowflake_connector.py | 14 +++++--------- toucan_connectors/snowflake_common.py | 4 ++-- .../snowflake_oauth2/snowflake_oauth2_connector.py | 2 +- toucan_connectors/soap/soap_connector.py | 2 +- toucan_connectors/toucan_connector.py | 2 +- 33 files changed, 73 insertions(+), 78 deletions(-) diff --git a/toucan_connectors/anaplan/anaplan_connector.py b/toucan_connectors/anaplan/anaplan_connector.py index 7e66a5f85..06cc4e0a3 100644 --- a/toucan_connectors/anaplan/anaplan_connector.py +++ b/toucan_connectors/anaplan/anaplan_connector.py @@ -4,16 +4,16 @@ import pandas as pd import pydantic import requests -from pydantic import StringConstraints, Field, create_model +from pydantic import Field, StringConstraints, create_model +from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated _ID_SEPARATOR = ' - ' diff --git a/toucan_connectors/awsathena/awsathena_connector.py b/toucan_connectors/awsathena/awsathena_connector.py index e0dcca21b..1a3d6c10d 100644 --- a/toucan_connectors/awsathena/awsathena_connector.py +++ b/toucan_connectors/awsathena/awsathena_connector.py @@ -4,22 +4,22 @@ import boto3 import pandas as pd from cached_property import cached_property_with_ttl -from pydantic import StringConstraints, ConfigDict, Field, create_model +from pydantic import ConfigDict, Field, StringConstraints, create_model +from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus, apply_query_parameters, sanitize_query from toucan_connectors.pagination import build_pagination_info from toucan_connectors.pandas_translator import PandasConditionTranslator from toucan_connectors.toucan_connector import ( - PlainJsonSecretStr, DataSlice, DataStats, DiscoverableConnector, + PlainJsonSecretStr, TableInfo, ToucanConnector, ToucanDataSource, strlist_to_enum, ) -from typing_extensions import Annotated class AwsathenaDataSource(ToucanDataSource): diff --git a/toucan_connectors/azure_mssql/azure_mssql_connector.py b/toucan_connectors/azure_mssql/azure_mssql_connector.py index 57ed8482a..81c335e2c 100644 --- a/toucan_connectors/azure_mssql/azure_mssql_connector.py +++ b/toucan_connectors/azure_mssql/azure_mssql_connector.py @@ -2,11 +2,11 @@ import pandas as pd import pyodbc -from pydantic import StringConstraints, Field +from pydantic import Field, StringConstraints +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr -from typing_extensions import Annotated +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource CLOUD_HOST = 'database.windows.net' diff --git a/toucan_connectors/clickhouse/clickhouse_connector.py b/toucan_connectors/clickhouse/clickhouse_connector.py index 01122ae4c..0f2481f83 100644 --- a/toucan_connectors/clickhouse/clickhouse_connector.py +++ b/toucan_connectors/clickhouse/clickhouse_connector.py @@ -1,18 +1,18 @@ from contextlib import suppress -from typing import Any, Dict, Type +from typing import Any import clickhouse_driver -from pydantic import StringConstraints, Field, create_model +from pydantic import Field, StringConstraints, create_model from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated class ClickhouseDataSource(ToucanDataSource): diff --git a/toucan_connectors/databricks/databricks_connector.py b/toucan_connectors/databricks/databricks_connector.py index 7abb4ffea..ec1762cd8 100644 --- a/toucan_connectors/databricks/databricks_connector.py +++ b/toucan_connectors/databricks/databricks_connector.py @@ -4,12 +4,12 @@ import pandas as pd import pyodbc import requests -from pydantic import StringConstraints, Field +from pydantic import Field, StringConstraints from requests.auth import HTTPBasicAuth +from typing_extensions import Annotated from toucan_connectors.common import ClusterStartException, ConnectorStatus, pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr -from typing_extensions import Annotated +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource logger = logging.getLogger(__name__) diff --git a/toucan_connectors/elasticsearch/elasticsearch_connector.py b/toucan_connectors/elasticsearch/elasticsearch_connector.py index 21db5719a..885b0502a 100644 --- a/toucan_connectors/elasticsearch/elasticsearch_connector.py +++ b/toucan_connectors/elasticsearch/elasticsearch_connector.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field from toucan_connectors.common import nosql_apply_parameters_to_query -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource def _is_branch_list(val): diff --git a/toucan_connectors/google_adwords/google_adwords_connector.py b/toucan_connectors/google_adwords/google_adwords_connector.py index 1d7132ba4..aeab94ac2 100644 --- a/toucan_connectors/google_adwords/google_adwords_connector.py +++ b/toucan_connectors/google_adwords/google_adwords_connector.py @@ -2,13 +2,13 @@ import os from io import StringIO from pathlib import Path -from typing import Any, Dict, Optional, Type +from typing import Any, Dict, Optional import pandas as pd -from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode import requests from googleads import AdWordsClient, adwords, oauth2 from pydantic import Field, PrivateAttr +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from zeep.helpers import serialize_object from toucan_connectors.common import ConnectorStatus, HttpError diff --git a/toucan_connectors/google_analytics/google_analytics_connector.py b/toucan_connectors/google_analytics/google_analytics_connector.py index 522771e5e..45af698e7 100644 --- a/toucan_connectors/google_analytics/google_analytics_connector.py +++ b/toucan_connectors/google_analytics/google_analytics_connector.py @@ -3,7 +3,7 @@ import pandas as pd from apiclient.discovery import build from oauth2client.service_account import ServiceAccountCredentials -from pydantic import ConfigDict, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from toucan_connectors.common import nosql_apply_parameters_to_query from toucan_connectors.google_credentials import GoogleCredentials diff --git a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py index 6fb8dcb58..846a3e836 100644 --- a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py +++ b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py @@ -1,10 +1,10 @@ import pandas as pd import pymysql -from pydantic import StringConstraints, Field +from pydantic import Field, StringConstraints +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr -from typing_extensions import Annotated +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource class GoogleCloudMySQLDataSource(ToucanDataSource): diff --git a/toucan_connectors/google_credentials.py b/toucan_connectors/google_credentials.py index cc9ee1314..a42e80a80 100644 --- a/toucan_connectors/google_credentials.py +++ b/toucan_connectors/google_credentials.py @@ -1,4 +1,4 @@ -from pydantic import field_validator, BaseModel, Field, HttpUrl +from pydantic import BaseModel, Field, HttpUrl, field_validator CREDENTIALS_INFO_MESSAGE = ( 'This information is provided in your ' diff --git a/toucan_connectors/google_sheets/google_sheets_connector.py b/toucan_connectors/google_sheets/google_sheets_connector.py index b37111ef9..80f24442a 100644 --- a/toucan_connectors/google_sheets/google_sheets_connector.py +++ b/toucan_connectors/google_sheets/google_sheets_connector.py @@ -1,6 +1,6 @@ from contextlib import suppress from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Type +from typing import Any, Callable, List, Optional import numpy as np import pandas as pd @@ -13,10 +13,10 @@ from toucan_connectors.common import ConnectorStatus from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) diff --git a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py index 4b9db665a..3cacea0dc 100644 --- a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py +++ b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py @@ -5,7 +5,7 @@ import os from contextlib import suppress from pathlib import Path -from typing import Any, Dict, List, Optional, Type +from typing import Any, List, Optional import pandas as pd from aiohttp import ClientSession diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index 26690baf6..88ced0c25 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -1,6 +1,6 @@ import json from enum import Enum -from typing import Any, Dict, List, Type, Union +from typing import Any, List, Union from xml.etree.ElementTree import ParseError, fromstring, tostring import pandas as pd diff --git a/toucan_connectors/hubspot_private_app/hubspot_connector.py b/toucan_connectors/hubspot_private_app/hubspot_connector.py index 80d906746..fcb8ec160 100644 --- a/toucan_connectors/hubspot_private_app/hubspot_connector.py +++ b/toucan_connectors/hubspot_private_app/hubspot_connector.py @@ -4,14 +4,14 @@ import pandas as pd from hubspot import HubSpot # type:ignore[import] -from pydantic import ConfigDict, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from toucan_connectors.pagination import build_pagination_info from toucan_connectors.toucan_connector import ( DataSlice, + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, - PlainJsonSecretStr, ) diff --git a/toucan_connectors/linkedinads/linkedinads_connector.py b/toucan_connectors/linkedinads/linkedinads_connector.py index 282ad1478..9aaa5c061 100644 --- a/toucan_connectors/linkedinads/linkedinads_connector.py +++ b/toucan_connectors/linkedinads/linkedinads_connector.py @@ -3,13 +3,13 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Dict, Optional, Type +from typing import Any, Optional import dateutil.parser import pandas as pd -from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode import requests from pydantic import Field, PrivateAttr +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from toucan_data_sdk.utils.postprocess.json_to_table import json_to_table from toucan_connectors.common import ConnectorStatus, HttpError diff --git a/toucan_connectors/micro_strategy/micro_strategy_connector.py b/toucan_connectors/micro_strategy/micro_strategy_connector.py index 10eda4e1a..b8881dc54 100644 --- a/toucan_connectors/micro_strategy/micro_strategy_connector.py +++ b/toucan_connectors/micro_strategy/micro_strategy_connector.py @@ -5,7 +5,7 @@ from pydantic import Field, HttpUrl from toucan_connectors.common import nosql_apply_parameters_to_query -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource from .client import Client from .data import ( diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 6b33c4ebf..9aacf2fdc 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -5,15 +5,15 @@ import pymongo from bson.son import SON from cached_property import cached_property -from pydantic import ConfigDict, Field, create_model, model_validator, validator +from pydantic import ConfigDict, Field, create_model, model_validator from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query from toucan_connectors.json_wrapper import JsonWrapper from toucan_connectors.mongo.mongo_translator import MongoConditionTranslator from toucan_connectors.pagination import build_pagination_info from toucan_connectors.toucan_connector import ( - PlainJsonSecretStr, DataSlice, + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, UnavailableVersion, diff --git a/toucan_connectors/mssql/mssql_connector.py b/toucan_connectors/mssql/mssql_connector.py index 8453b8f47..eb26aab47 100644 --- a/toucan_connectors/mssql/mssql_connector.py +++ b/toucan_connectors/mssql/mssql_connector.py @@ -1,16 +1,16 @@ from contextlib import suppress import pyodbc -from pydantic import StringConstraints, Field, create_model +from pydantic import Field, StringConstraints, create_model +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated class MSSQLDataSource(ToucanDataSource): diff --git a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py index 9a0db3192..e63987abd 100644 --- a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py +++ b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py @@ -2,16 +2,16 @@ from contextlib import suppress import pyodbc -from pydantic import StringConstraints, Field, create_model +from pydantic import Field, StringConstraints, create_model +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated class MSSQLDataSource(ToucanDataSource): diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index c85785b89..1e9daa8ae 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -7,22 +7,22 @@ import pandas as pd import pymysql from cached_property import cached_property_with_ttl -from pydantic import StringConstraints, ConfigDict, Field, create_model, model_validator, validator +from pydantic import ConfigDict, Field, StringConstraints, create_model, model_validator from pymysql.constants import CR, ER +from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus, pandas_read_sql from toucan_connectors.toucan_connector import ( DiscoverableConnector, + PlainJsonSecretStr, TableInfo, ToucanConnector, ToucanDataSource, UnavailableVersion, VersionableEngineConnector, strlist_to_enum, - PlainJsonSecretStr, ) from toucan_connectors.utils.pem import sanitize_spaces_pem -from typing_extensions import Annotated def handle_date_0(df: pd.DataFrame) -> pd.DataFrame: diff --git a/toucan_connectors/net_explorer/net_explorer_connector.py b/toucan_connectors/net_explorer/net_explorer_connector.py index 6d022636e..949526fd6 100755 --- a/toucan_connectors/net_explorer/net_explorer_connector.py +++ b/toucan_connectors/net_explorer/net_explorer_connector.py @@ -8,7 +8,7 @@ from pydantic import Field from toucan_connectors.common import ConnectorStatus -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource class NetExplorerDataSource(ToucanDataSource): diff --git a/toucan_connectors/odbc/odbc_connector.py b/toucan_connectors/odbc/odbc_connector.py index a76c87dca..664478ebf 100644 --- a/toucan_connectors/odbc/odbc_connector.py +++ b/toucan_connectors/odbc/odbc_connector.py @@ -1,10 +1,10 @@ import pandas as pd import pyodbc -from pydantic import StringConstraints, Field +from pydantic import Field, StringConstraints +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource -from typing_extensions import Annotated class OdbcDataSource(ToucanDataSource): diff --git a/toucan_connectors/one_drive/one_drive_connector.py b/toucan_connectors/one_drive/one_drive_connector.py index de23ef662..b77acc606 100755 --- a/toucan_connectors/one_drive/one_drive_connector.py +++ b/toucan_connectors/one_drive/one_drive_connector.py @@ -11,7 +11,7 @@ OAuth2Connector, OAuth2ConnectorConfig, ) -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource class NotFoundError(Exception): diff --git a/toucan_connectors/oracle_sql/oracle_sql_connector.py b/toucan_connectors/oracle_sql/oracle_sql_connector.py index 5fbfcb729..03efd1c71 100644 --- a/toucan_connectors/oracle_sql/oracle_sql_connector.py +++ b/toucan_connectors/oracle_sql/oracle_sql_connector.py @@ -2,16 +2,16 @@ import cx_Oracle import pandas as pd -from pydantic import StringConstraints, Field, create_model +from pydantic import Field, StringConstraints, create_model +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ( + PlainJsonSecretStr, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated class OracleSQLDataSource(ToucanDataSource): diff --git a/toucan_connectors/peakina/peakina_connector.py b/toucan_connectors/peakina/peakina_connector.py index 44e4144ce..4f8f8b38f 100644 --- a/toucan_connectors/peakina/peakina_connector.py +++ b/toucan_connectors/peakina/peakina_connector.py @@ -2,9 +2,9 @@ import pandas as pd from peakina.datasource import DataSource +from pydantic import ConfigDict from toucan_connectors.toucan_connector import ToucanConnector -from pydantic import ConfigDict class PeakinaDataSource(DataSource): diff --git a/toucan_connectors/postgres/postgresql_connector.py b/toucan_connectors/postgres/postgresql_connector.py index 4081c62ac..6768328ea 100644 --- a/toucan_connectors/postgres/postgresql_connector.py +++ b/toucan_connectors/postgres/postgresql_connector.py @@ -2,21 +2,21 @@ from typing import Dict, List, Optional import psycopg2 as pgsql -from pydantic import StringConstraints, Field, create_model +from pydantic import Field, StringConstraints, create_model +from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus, pandas_read_sql from toucan_connectors.postgres.utils import build_database_model_extraction_query, types from toucan_connectors.toucan_connector import ( DiscoverableConnector, + PlainJsonSecretStr, TableInfo, ToucanConnector, ToucanDataSource, UnavailableVersion, VersionableEngineConnector, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated DEFAULT_DATABASE = 'postgres' diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index d6f47a54c..3ec55e19a 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -6,17 +6,17 @@ from typing import Any import pandas as pd -from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode import redshift_connector from pydantic import ( ConfigDict, - field_validator, - StringConstraints, Field, + StringConstraints, create_model, + field_validator, root_validator, ) -from pydantic.types import constr +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode +from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus from toucan_connectors.pagination import build_pagination_info @@ -25,13 +25,12 @@ from toucan_connectors.toucan_connector import ( DataSlice, DiscoverableConnector, + PlainJsonSecretStr, TableInfo, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) -from typing_extensions import Annotated TABLE_QUERY = """SELECT DISTINCT tablename FROM pg_table_def WHERE schemaname = 'public';""" diff --git a/toucan_connectors/sap_hana/sap_hana_connector.py b/toucan_connectors/sap_hana/sap_hana_connector.py index 91ca805a0..e51c38e52 100644 --- a/toucan_connectors/sap_hana/sap_hana_connector.py +++ b/toucan_connectors/sap_hana/sap_hana_connector.py @@ -1,9 +1,9 @@ import pyhdb -from pydantic import StringConstraints, Field +from pydantic import Field, StringConstraints +from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql -from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, PlainJsonSecretStr -from typing_extensions import Annotated +from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource class SapHanaDataSource(ToucanDataSource): diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 94efa5fe6..f58e6211a 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -2,20 +2,15 @@ from contextlib import contextmanager, suppress from datetime import datetime from enum import Enum -from typing import Any, ContextManager, Generator, Literal, Type, overload +from typing import Any, ContextManager, Generator, Literal, overload import jwt import pandas as pd -from pydantic.json_schema import ( - DEFAULT_REF_TEMPLATE, - GenerateJsonSchema, - JsonSchemaMode, - model_json_schema, -) import requests import snowflake from jinja2 import Template from pydantic import Field, create_model +from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from snowflake import connector as sf_connector from snowflake.connector import SnowflakeConnection from snowflake.connector.cursor import DictCursor as SfDictCursor @@ -31,11 +26,11 @@ Category, DataSlice, DiscoverableConnector, + PlainJsonSecretStr, TableInfo, ToucanConnector, ToucanDataSource, strlist_to_enum, - PlainJsonSecretStr, ) logger = logging.getLogger(__name__) @@ -181,7 +176,8 @@ def model_json_schema( cls, by_alias: bool = True, ref_template: str = DEFAULT_REF_TEMPLATE, - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + # This confuses mypy because of the type field + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, # type:ignore[valid-type] mode: JsonSchemaMode = 'validation', ) -> dict[str, Any]: schema = super().model_json_schema( diff --git a/toucan_connectors/snowflake_common.py b/toucan_connectors/snowflake_common.py index 69a406040..cca30b936 100644 --- a/toucan_connectors/snowflake_common.py +++ b/toucan_connectors/snowflake_common.py @@ -5,14 +5,14 @@ from typing import Any, Dict, List, Optional import pandas as pd -from pydantic import StringConstraints, Field +from pydantic import Field, StringConstraints from snowflake.connector import DictCursor, SnowflakeConnection +from typing_extensions import Annotated from toucan_connectors.pagination import build_pagination_info from toucan_connectors.query_manager import QueryManager from toucan_connectors.sql_query_helper import SqlQueryHelper from toucan_connectors.toucan_connector import DataSlice, DataStats, QueryMetadata, ToucanDataSource -from typing_extensions import Annotated type_code_mapping = { 0: 'float', diff --git a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py index 3a8024e0a..83c903ed0 100644 --- a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py +++ b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py @@ -26,9 +26,9 @@ Category, DataSlice, DiscoverableConnector, + PlainJsonSecretStr, ToucanConnector, strlist_to_enum, - PlainJsonSecretStr, ) logger = logging.getLogger(__name__) diff --git a/toucan_connectors/soap/soap_connector.py b/toucan_connectors/soap/soap_connector.py index fbfa758f2..a795c7afb 100644 --- a/toucan_connectors/soap/soap_connector.py +++ b/toucan_connectors/soap/soap_connector.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Type +from typing import Any import pandas as pd from pydantic import Field, create_model diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index e45b79782..2f1a24be1 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -12,7 +12,7 @@ import pandas as pd import tenacity as tny -from pydantic import ConfigDict, BaseModel, Field, PlainSerializer, SecretBytes, SecretStr +from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, SecretStr from toucan_connectors.common import ( ConnectorStatus, From 22da7b4335cc434876af13973436b3696ed246c7 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 12:06:36 +0200 Subject: [PATCH 08/27] fix: type annotation for oauth2_version Signed-off-by: Luka Peschke --- toucan_connectors/facebook_ads/facebook_ads_connector.py | 2 +- toucan_connectors/github/github_connector.py | 2 +- toucan_connectors/google_adwords/google_adwords_connector.py | 2 +- toucan_connectors/google_sheets_2/google_sheets_2_connector.py | 2 +- toucan_connectors/hubspot/hubspot_connector.py | 2 +- toucan_connectors/linkedinads/linkedinads_connector.py | 2 +- toucan_connectors/one_drive/one_drive_connector.py | 2 +- toucan_connectors/salesforce/salesforce_connector.py | 2 +- .../snowflake_oauth2/snowflake_oauth2_connector.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/toucan_connectors/facebook_ads/facebook_ads_connector.py b/toucan_connectors/facebook_ads/facebook_ads_connector.py index d534c45ad..839b28dc7 100644 --- a/toucan_connectors/facebook_ads/facebook_ads_connector.py +++ b/toucan_connectors/facebook_ads/facebook_ads_connector.py @@ -81,7 +81,7 @@ class FacebookAdsConnector(ToucanConnector): auth_flow_id: Optional[str] = None _oauth_trigger = 'instance' data_source_model: FacebookAdsDataSource - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) _oauth2_connector: OAuth2Connector = PrivateAttr() def __init__(self, **kwargs) -> None: diff --git a/toucan_connectors/github/github_connector.py b/toucan_connectors/github/github_connector.py index 951b7f4d6..624e99f5e 100644 --- a/toucan_connectors/github/github_connector.py +++ b/toucan_connectors/github/github_connector.py @@ -82,7 +82,7 @@ class GithubDataSource(ToucanDataSource): description='Max Number of entities such as teams and repositories to extract', ) _oauth_trigger = 'instance' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) @classmethod def get_form(cls, connector: 'GithubConnector', current_config, **kwargs): diff --git a/toucan_connectors/google_adwords/google_adwords_connector.py b/toucan_connectors/google_adwords/google_adwords_connector.py index aeab94ac2..bd6815f20 100644 --- a/toucan_connectors/google_adwords/google_adwords_connector.py +++ b/toucan_connectors/google_adwords/google_adwords_connector.py @@ -96,7 +96,7 @@ class GoogleAdwordsConnector(ToucanConnector): developer_token: str = None client_customer_id: str = None _oauth_trigger = 'instance' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) _oauth2_connector: OAuth2Connector = PrivateAttr() @staticmethod diff --git a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py index 3cacea0dc..f7de7b84d 100644 --- a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py +++ b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py @@ -110,7 +110,7 @@ class GoogleSheets2Connector(ToucanConnector): _auth_flow = 'oauth2' _oauth_trigger = 'instance' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) auth_flow_id: Optional[str] = None # TODO: turn into a class property diff --git a/toucan_connectors/hubspot/hubspot_connector.py b/toucan_connectors/hubspot/hubspot_connector.py index 7bd2d45a2..53c2f5f2f 100644 --- a/toucan_connectors/hubspot/hubspot_connector.py +++ b/toucan_connectors/hubspot/hubspot_connector.py @@ -72,7 +72,7 @@ class HubspotConnector(ToucanConnector): _auth_flow = 'oauth2' auth_flow_id: Optional[str] = None _oauth_trigger = 'instance' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) data_source_model: HubspotDataSource _oauth2_connector: OAuth2Connector = PrivateAttr() diff --git a/toucan_connectors/linkedinads/linkedinads_connector.py b/toucan_connectors/linkedinads/linkedinads_connector.py index 9aaa5c061..77bb71edd 100644 --- a/toucan_connectors/linkedinads/linkedinads_connector.py +++ b/toucan_connectors/linkedinads/linkedinads_connector.py @@ -116,7 +116,7 @@ class LinkedinadsConnector(ToucanConnector): description='You can provide a custom template that will be used for every HTTP request', ) _oauth_trigger = 'instance' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) _oauth2_connector: OAuth2Connector = PrivateAttr() @staticmethod diff --git a/toucan_connectors/one_drive/one_drive_connector.py b/toucan_connectors/one_drive/one_drive_connector.py index b77acc606..7b3efdb2b 100755 --- a/toucan_connectors/one_drive/one_drive_connector.py +++ b/toucan_connectors/one_drive/one_drive_connector.py @@ -76,7 +76,7 @@ class OneDriveConnector(ToucanConnector): _auth_flow = 'oauth2' _oauth_trigger = 'connector' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) auth_flow_id: Optional[str] = None authorization_url: str = Field(None, **{'ui.hidden': True}) diff --git a/toucan_connectors/salesforce/salesforce_connector.py b/toucan_connectors/salesforce/salesforce_connector.py index 2c78aa8b8..96f18aba5 100644 --- a/toucan_connectors/salesforce/salesforce_connector.py +++ b/toucan_connectors/salesforce/salesforce_connector.py @@ -59,7 +59,7 @@ class SalesforceConnector(ToucanConnector): description='Baseroute URL of the salesforces instance to query (without the trailing slash)', ) _oauth_trigger = 'instance' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) _oauth2_connector: OAuth2Connector = PrivateAttr() @staticmethod diff --git a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py index 83c903ed0..1e510e4b7 100644 --- a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py +++ b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py @@ -82,7 +82,7 @@ class SnowflakeoAuth2Connector(ToucanConnector): auth_flow_id: str = Field(None, **{'ui.hidden': True}) _auth_flow = 'oauth2' _oauth_trigger = 'connector' - oauth2_version = Field('1', **{'ui.hidden': True}) + oauth2_version: str = Field('1', **{'ui.hidden': True}) redirect_uri: str = Field(None, **{'ui.hidden': True}) _oauth2_connector: OAuth2Connector = PrivateAttr() From fdc3164535d6fecd26d29e3f4ddf5b0f59c5e4c4 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 12:09:04 +0200 Subject: [PATCH 09/27] fix: replace root_validator with model_validator Signed-off-by: Luka Peschke --- .../redshift/redshift_database_connector.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index 3ec55e19a..c94f19c99 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -13,7 +13,7 @@ StringConstraints, create_model, field_validator, - root_validator, + model_validator, ) from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode from typing_extensions import Annotated @@ -181,29 +181,22 @@ def available_dbs(self) -> list[str]: def host_validator(cls, v): return re.sub(r'^https?://', '', v) - @root_validator - def check_requirements(cls, values): - mode = values.get('authentication_method') - if mode == AuthenticationMethod.DB_CREDENTIALS.value: + @model_validator(mode='after') + def check_requirements(self): + if self.authentication_method == AuthenticationMethod.DB_CREDENTIALS.value: # TODO: Partial check due to missing context in some operations (Missing: password) - user = values.get('user') - if user is None: + if self.user is None: raise ValueError(AuthenticationMethodError.DB_CREDENTIALS.value) - elif mode == AuthenticationMethod.AWS_CREDENTIALS.value: + elif self.authentication_method == AuthenticationMethod.AWS_CREDENTIALS.value: # TODO: Partial check due to missing context in some operations (Missing: secret_access_key) - access_key_id, db_user = ( - values.get('access_key_id'), - values.get('db_user'), - ) - if access_key_id is None or db_user is None: + if self.access_key_id is None or self.db_user is None: raise ValueError(AuthenticationMethodError.AWS_CREDENTIALS.value) - elif mode == AuthenticationMethod.AWS_PROFILE.value: - profile, db_user = (values.get('profile'), values.get('db_user')) - if profile is None or db_user is None: + elif self.authentication_method == AuthenticationMethod.AWS_PROFILE.value: + if self.profile is None or self.db_user is None: raise ValueError(AuthenticationMethodError.AWS_PROFILE.value) else: raise ValueError(AuthenticationMethodError.UNKNOWN.value) - return values + return self def _get_connection_params(self, database) -> dict[str, Any]: con_params = dict( From 6a198878fed3f9f3900ca5ae99d4b99bf63085a7 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 14:24:42 +0200 Subject: [PATCH 10/27] fix typing Signed-off-by: Luka Peschke --- .../hubspot_private_app/hubspot_connector.py | 2 +- toucan_connectors/peakina/peakina_connector.py | 2 +- toucan_connectors/snowflake/snowflake_connector.py | 2 +- toucan_connectors/toucan_connector.py | 11 +++-------- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/toucan_connectors/hubspot_private_app/hubspot_connector.py b/toucan_connectors/hubspot_private_app/hubspot_connector.py index fcb8ec160..694afafce 100644 --- a/toucan_connectors/hubspot_private_app/hubspot_connector.py +++ b/toucan_connectors/hubspot_private_app/hubspot_connector.py @@ -92,7 +92,7 @@ def _page_api_for(api: _Api, dataset: HubspotDataset) -> _PageApi: class HubspotConnector(ToucanConnector): - data_source_model: HubspotDataSource + data_source_model: type[HubspotDataSource] = HubspotDataSource access_token: PlainJsonSecretStr = Field( ..., description='An API key for the target private app' ) diff --git a/toucan_connectors/peakina/peakina_connector.py b/toucan_connectors/peakina/peakina_connector.py index 4f8f8b38f..8cd4a2d5c 100644 --- a/toucan_connectors/peakina/peakina_connector.py +++ b/toucan_connectors/peakina/peakina_connector.py @@ -15,7 +15,7 @@ def __init__(self, **data: Any) -> None: class PeakinaConnector(ToucanConnector): - data_source_model: PeakinaDataSource + data_source_model: type[PeakinaDataSource] = PeakinaDataSource def _retrieve_data(self, data_source: PeakinaDataSource) -> pd.DataFrame: return data_source.get_df() diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index f58e6211a..495af2d78 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -134,7 +134,7 @@ class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConne sso_credentials_keeper: Any = None user_tokens_keeper: Any = None - data_source_model: SnowflakeDataSource + data_source_model: type[SnowflakeDataSource] = SnowflakeDataSource account: str = Field( ..., diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index 2f1a24be1..617720377 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -283,10 +283,11 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): the `_retry_on` class attribute in your concrete connector class. """ + data_source_model: type[DS] name: str = Field(...) retry_policy: RetryPolicy | None = RetryPolicy() _retry_on: Iterable[Type[BaseException]] = () - type: str | None = None + type: str secrets_storage_version: str = Field('1', **_UI_HIDDEN) # type:ignore[pydantic-field] # Default ttl for all connector's queries (overridable at the data_source level) @@ -303,13 +304,7 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): @classmethod def __init_subclass__(cls): - try: - cls.data_source_model = cls.__fields__.pop('data_source_model').type_ - cls.logger = logging.getLogger(cls.__name__) - except KeyError as e: - raise TypeError(f'{cls.__name__} has no {e} attribute.') - if 'bearer_integration' in cls.__fields__: - cls.bearer_integration = cls.__fields__['bearer_integration'].default + cls.logger = logging.getLogger(cls.__name__) def bearer_oauth_get_endpoint( self, From a1212261213ad620c5a85a781013242eaa43e748 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 16:51:20 +0200 Subject: [PATCH 11/27] refactor: Make data_source_model an attribute of __init_subclass__ Signed-off-by: Luka Peschke --- .../adobe_analytics_connector.py | 4 +- .../anaplan/anaplan_connector.py | 7 ++- .../awsathena/awsathena_connector.py | 6 +- .../azure_mssql/azure_mssql_connector.py | 4 +- .../clickhouse/clickhouse_connector.py | 3 +- .../databricks/databricks_connector.py | 3 +- .../dataiku/dataiku_connector.py | 4 +- .../elasticsearch/elasticsearch_connector.py | 3 +- .../facebook_ads/facebook_ads_connector.py | 3 +- .../facebook_insights_connector.py | 4 +- toucan_connectors/github/github_connector.py | 3 +- .../google_adwords_connector.py | 3 +- .../google_analytics_connector.py | 4 +- .../google_big_query_connector.py | 6 +- .../google_cloud_mysql_connector.py | 4 +- .../google_my_business_connector.py | 4 +- .../google_sheets/google_sheets_connector.py | 4 +- .../google_sheets_2_connector.py | 4 +- .../google_spreadsheet_connector.py | 4 +- .../http_api/http_api_connector.py | 3 +- .../hubspot/hubspot_connector.py | 3 +- .../hubspot_private_app/hubspot_connector.py | 3 +- .../linkedinads/linkedinads_connector.py | 3 +- .../micro_strategy_connector.py | 8 +-- toucan_connectors/mongo/mongo_connector.py | 6 +- toucan_connectors/mssql/mssql_connector.py | 4 +- .../mssql_TLSv1_0/mssql_connector.py | 4 +- toucan_connectors/mysql/mysql_connector.py | 9 ++- .../net_explorer/net_explorer_connector.py | 9 +-- toucan_connectors/odata/odata_connector.py | 4 +- toucan_connectors/odbc/odbc_connector.py | 4 +- .../one_drive/one_drive_connector.py | 4 +- .../oracle_sql/oracle_sql_connector.py | 4 +- .../peakina/peakina_connector.py | 4 +- .../postgres/postgresql_connector.py | 9 ++- .../redshift/redshift_database_connector.py | 5 +- .../salesforce/salesforce_connector.py | 3 +- .../sap_hana/sap_hana_connector.py | 4 +- .../snowflake/snowflake_connector.py | 8 ++- toucan_connectors/snowflake_common.py | 2 +- .../snowflake_oauth2_connector.py | 3 +- toucan_connectors/soap/soap_connector.py | 3 +- toucan_connectors/toucan_connector.py | 57 ++++++++++++++++--- .../toucan_toco/toucan_toco_connector.py | 4 +- toucan_connectors/trello/trello_connector.py | 8 +-- .../wootric/wootric_connector.py | 4 +- 46 files changed, 125 insertions(+), 134 deletions(-) diff --git a/toucan_connectors/adobe_analytics/adobe_analytics_connector.py b/toucan_connectors/adobe_analytics/adobe_analytics_connector.py index 1411bfc84..48fcec72a 100644 --- a/toucan_connectors/adobe_analytics/adobe_analytics_connector.py +++ b/toucan_connectors/adobe_analytics/adobe_analytics_connector.py @@ -43,14 +43,12 @@ def report_definition(self): ) -class AdobeAnalyticsConnector(ToucanConnector): +class AdobeAnalyticsConnector(ToucanConnector, data_source_model=AdobeAnalyticsDataSource): """ Adobe Analytics Connector using Adobe Analytics' REST API v1.4. It provides a high-level interfaces for reporting queries (including Data Warehouse requests). """ - data_source_model: AdobeAnalyticsDataSource - username: str password: str endpoint: str = Client.DEFAULT_ENDPOINT diff --git a/toucan_connectors/anaplan/anaplan_connector.py b/toucan_connectors/anaplan/anaplan_connector.py index 06cc4e0a3..c885911c9 100644 --- a/toucan_connectors/anaplan/anaplan_connector.py +++ b/toucan_connectors/anaplan/anaplan_connector.py @@ -4,7 +4,7 @@ import pandas as pd import pydantic import requests -from pydantic import Field, StringConstraints, create_model +from pydantic import ConfigDict, Field, StringConstraints, create_model from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus @@ -35,6 +35,8 @@ class AnaplanDataSource(ToucanDataSource): ) workspace_id: str = Field(..., description='The ID of the workspace you want to query') + model_config = ConfigDict(protected_namespaces=()) + @pydantic.validator('model_id', 'view_id', 'workspace_id') def _sanitize_id(cls, value: str) -> str: return _sanitize_id(value) @@ -96,8 +98,7 @@ class AnaplanAuthError(AnaplanError): _ANAPLAN_API_BASEROUTE = 'https://api.anaplan.com/2/0' -class AnaplanConnector(ToucanConnector): - data_source_model: AnaplanDataSource +class AnaplanConnector(ToucanConnector, data_source_model=AnaplanDataSource): username: str password: Optional[PlainJsonSecretStr] = None diff --git a/toucan_connectors/awsathena/awsathena_connector.py b/toucan_connectors/awsathena/awsathena_connector.py index 1a3d6c10d..7d8f2e5a6 100644 --- a/toucan_connectors/awsathena/awsathena_connector.py +++ b/toucan_connectors/awsathena/awsathena_connector.py @@ -68,9 +68,9 @@ def athena_variable_transformer(variable: str): return f':{variable};' -class AwsathenaConnector(ToucanConnector, DiscoverableConnector): - data_source_model: AwsathenaDataSource - +class AwsathenaConnector( + ToucanConnector, DiscoverableConnector, data_source_model=AwsathenaDataSource +): name: str = Field(..., description='Your AWS Athena connector name') s3_output_bucket: str = Field( diff --git a/toucan_connectors/azure_mssql/azure_mssql_connector.py b/toucan_connectors/azure_mssql/azure_mssql_connector.py index 81c335e2c..d89783033 100644 --- a/toucan_connectors/azure_mssql/azure_mssql_connector.py +++ b/toucan_connectors/azure_mssql/azure_mssql_connector.py @@ -18,13 +18,11 @@ class AzureMSSQLDataSource(ToucanDataSource): ) -class AzureMSSQLConnector(ToucanConnector): +class AzureMSSQLConnector(ToucanConnector, data_source_model=AzureMSSQLDataSource): """ Import data from Microsoft Azure SQL Server. """ - data_source_model: AzureMSSQLDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/clickhouse/clickhouse_connector.py b/toucan_connectors/clickhouse/clickhouse_connector.py index 0f2481f83..611886ce4 100644 --- a/toucan_connectors/clickhouse/clickhouse_connector.py +++ b/toucan_connectors/clickhouse/clickhouse_connector.py @@ -96,12 +96,11 @@ def get_form(cls, connector: 'ClickhouseConnector', current_config): return create_model('FormSchema', **constraints, __base__=cls).schema() -class ClickhouseConnector(ToucanConnector): +class ClickhouseConnector(ToucanConnector, data_source_model=ClickhouseDataSource): """ Import data from Clickhouse. """ - data_source_model: ClickhouseDataSource host: str = Field( None, description='Use this parameter if you have an IP address. ' diff --git a/toucan_connectors/databricks/databricks_connector.py b/toucan_connectors/databricks/databricks_connector.py index ec1762cd8..afa4523e1 100644 --- a/toucan_connectors/databricks/databricks_connector.py +++ b/toucan_connectors/databricks/databricks_connector.py @@ -22,8 +22,7 @@ class DatabricksDataSource(ToucanDataSource): ) -class DatabricksConnector(ToucanConnector): - data_source_model: DatabricksDataSource +class DatabricksConnector(ToucanConnector, data_source_model=DatabricksDataSource): host: str = Field( ..., description='The listening address of your databricks cluster', diff --git a/toucan_connectors/dataiku/dataiku_connector.py b/toucan_connectors/dataiku/dataiku_connector.py index ef53732c7..cf5f5cc0b 100644 --- a/toucan_connectors/dataiku/dataiku_connector.py +++ b/toucan_connectors/dataiku/dataiku_connector.py @@ -11,13 +11,13 @@ class DataikuDataSource(ToucanDataSource): dataset: str -class DataikuConnector(ToucanConnector): +class DataikuConnector(ToucanConnector, data_source_model=DataikuDataSource): """ This is a basic connector for [Dataiku](https://www.dataiku.com/) using their [DSS API](https://doc.dataiku.com/dss/2.0/api/index.html). """ - data_source_model: DataikuDataSource + data_source_model: type[DataikuDataSource] = DataikuDataSource host: str = Field( ..., diff --git a/toucan_connectors/elasticsearch/elasticsearch_connector.py b/toucan_connectors/elasticsearch/elasticsearch_connector.py index 885b0502a..a97c01734 100644 --- a/toucan_connectors/elasticsearch/elasticsearch_connector.py +++ b/toucan_connectors/elasticsearch/elasticsearch_connector.py @@ -117,8 +117,7 @@ class ElasticsearchDataSource(ToucanDataSource): body: Union[dict, list] -class ElasticsearchConnector(ToucanConnector): - data_source_model: ElasticsearchDataSource +class ElasticsearchConnector(ToucanConnector, data_source_model=ElasticsearchDataSource): hosts: List[ElasticsearchHost] def _retrieve_data(self, data_source: ElasticsearchDataSource) -> pd.DataFrame: diff --git a/toucan_connectors/facebook_ads/facebook_ads_connector.py b/toucan_connectors/facebook_ads/facebook_ads_connector.py index 839b28dc7..f08a588e4 100644 --- a/toucan_connectors/facebook_ads/facebook_ads_connector.py +++ b/toucan_connectors/facebook_ads/facebook_ads_connector.py @@ -76,11 +76,10 @@ def determine_query_params(self) -> Dict[str, str]: return params -class FacebookAdsConnector(ToucanConnector): +class FacebookAdsConnector(ToucanConnector, data_source_model=FacebookAdsDataSource): _auth_flow = 'oauth2' auth_flow_id: Optional[str] = None _oauth_trigger = 'instance' - data_source_model: FacebookAdsDataSource oauth2_version: str = Field('1', **{'ui.hidden': True}) _oauth2_connector: OAuth2Connector = PrivateAttr() diff --git a/toucan_connectors/facebook_insights/facebook_insights_connector.py b/toucan_connectors/facebook_insights/facebook_insights_connector.py index 2ac764a4b..327449869 100644 --- a/toucan_connectors/facebook_insights/facebook_insights_connector.py +++ b/toucan_connectors/facebook_insights/facebook_insights_connector.py @@ -78,9 +78,7 @@ class FacebookInsightsDataSource(ToucanDataSource): date_preset: str = 'last_30d' -class FacebookInsightsConnector(ToucanConnector): - data_source_model: FacebookInsightsDataSource - +class FacebookInsightsConnector(ToucanConnector, data_source_model=FacebookInsightsDataSource): def _retrieve_data(self, data_source: FacebookInsightsDataSource) -> pd.DataFrame: """Return the concatenated insights for all pages. diff --git a/toucan_connectors/github/github_connector.py b/toucan_connectors/github/github_connector.py index 624e99f5e..5dea888db 100644 --- a/toucan_connectors/github/github_connector.py +++ b/toucan_connectors/github/github_connector.py @@ -95,10 +95,9 @@ def get_form(cls, connector: 'GithubConnector', current_config, **kwargs): return create_model('FormSchema', **constraints, __base__=cls).schema() -class GithubConnector(ToucanConnector): +class GithubConnector(ToucanConnector, data_source_model=GithubDataSource): _auth_flow = 'oauth2' auth_flow_id: Optional[str] = None - data_source_model: GithubDataSource _oauth_trigger = 'instance' _oauth2_connector: OAuth2Connector = PrivateAttr() diff --git a/toucan_connectors/google_adwords/google_adwords_connector.py b/toucan_connectors/google_adwords/google_adwords_connector.py index bd6815f20..60e074ca0 100644 --- a/toucan_connectors/google_adwords/google_adwords_connector.py +++ b/toucan_connectors/google_adwords/google_adwords_connector.py @@ -89,8 +89,7 @@ def model_json_schema( return schema -class GoogleAdwordsConnector(ToucanConnector): - data_source_model: GoogleAdwordsDataSource +class GoogleAdwordsConnector(ToucanConnector, data_source_model=GoogleAdwordsDataSource): _auth_flow = 'oauth2' auth_flow_id: Optional[str] = None developer_token: str = None diff --git a/toucan_connectors/google_analytics/google_analytics_connector.py b/toucan_connectors/google_analytics/google_analytics_connector.py index 45af698e7..8f77d1c80 100644 --- a/toucan_connectors/google_analytics/google_analytics_connector.py +++ b/toucan_connectors/google_analytics/google_analytics_connector.py @@ -157,9 +157,7 @@ class GoogleAnalyticsDataSource(ToucanDataSource): ) -class GoogleAnalyticsConnector(ToucanConnector): - data_source_model: GoogleAnalyticsDataSource - +class GoogleAnalyticsConnector(ToucanConnector, data_source_model=GoogleAnalyticsDataSource): credentials: GoogleCredentials = Field( ..., title='Google Credentials', diff --git a/toucan_connectors/google_big_query/google_big_query_connector.py b/toucan_connectors/google_big_query/google_big_query_connector.py index ed2ab3c5a..ed9ef897f 100644 --- a/toucan_connectors/google_big_query/google_big_query_connector.py +++ b/toucan_connectors/google_big_query/google_big_query_connector.py @@ -77,9 +77,9 @@ def _define_query_param(name: str, value: Any) -> BigQueryParam: return bigquery_helpers.scalar_to_query_parameter(value=value, name=name) -class GoogleBigQueryConnector(ToucanConnector, DiscoverableConnector): - data_source_model: GoogleBigQueryDataSource - +class GoogleBigQueryConnector( + ToucanConnector, DiscoverableConnector, data_source_model=GoogleBigQueryDataSource +): credentials: GoogleCredentials = Field( ..., title='Google Credentials', diff --git a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py index 846a3e836..571304b82 100644 --- a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py +++ b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py @@ -14,13 +14,11 @@ class GoogleCloudMySQLDataSource(ToucanDataSource): ) -class GoogleCloudMySQLConnector(ToucanConnector): +class GoogleCloudMySQLConnector(ToucanConnector, data_source_model=GoogleCloudMySQLDataSource): """ Import data from Google Cloud MySQL database. """ - data_source_model: GoogleCloudMySQLDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/google_my_business/google_my_business_connector.py b/toucan_connectors/google_my_business/google_my_business_connector.py index 3cc6b0a18..29913ec6b 100644 --- a/toucan_connectors/google_my_business/google_my_business_connector.py +++ b/toucan_connectors/google_my_business/google_my_business_connector.py @@ -38,9 +38,7 @@ class GoogleCredentials(BaseModel): client_secret: str -class GoogleMyBusinessConnector(ToucanConnector): - data_source_model: GoogleMyBusinessDataSource - +class GoogleMyBusinessConnector(ToucanConnector, data_source_model=GoogleMyBusinessDataSource): credentials: GoogleCredentials scopes: List[str] = ['https://www.googleapis.com/auth/business.manage'] diff --git a/toucan_connectors/google_sheets/google_sheets_connector.py b/toucan_connectors/google_sheets/google_sheets_connector.py index 80f24442a..02c73d9bb 100644 --- a/toucan_connectors/google_sheets/google_sheets_connector.py +++ b/toucan_connectors/google_sheets/google_sheets_connector.py @@ -73,7 +73,7 @@ def get_form(cls, connector: 'GoogleSheetsConnector', current_config, **kwargs): return create_model('FormSchema', **constraints, __base__=cls).schema() -class GoogleSheetsConnector(ToucanConnector): +class GoogleSheetsConnector(ToucanConnector, data_source_model=GoogleSheetsDataSource): """ This is a connector for [GoogleSheets](https://developers.google.com/sheets/api/reference/rest) @@ -81,8 +81,6 @@ class GoogleSheetsConnector(ToucanConnector): Not to be confused with the OAuth2 connector, which handles all the OAuth2 process byt itself! """ - data_source_model: GoogleSheetsDataSource - _auth_flow = 'managed_oauth2' _managed_oauth_service_id = 'google-sheets' _oauth_trigger = 'retrieve_token' diff --git a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py index f7de7b84d..b9d732357 100644 --- a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py +++ b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py @@ -103,11 +103,9 @@ def get_form(cls, connector: 'GoogleSheets2Connector', current_config, **kwargs) return create_model('FormSchema', **constraints, __base__=cls).schema() -class GoogleSheets2Connector(ToucanConnector): +class GoogleSheets2Connector(ToucanConnector, data_source_model=GoogleSheets2DataSource): """The Google Sheets connector.""" - data_source_model: GoogleSheets2DataSource - _auth_flow = 'oauth2' _oauth_trigger = 'instance' oauth2_version: str = Field('1', **{'ui.hidden': True}) diff --git a/toucan_connectors/google_spreadsheet/google_spreadsheet_connector.py b/toucan_connectors/google_spreadsheet/google_spreadsheet_connector.py index 3577b94d6..c1fac2f9d 100644 --- a/toucan_connectors/google_spreadsheet/google_spreadsheet_connector.py +++ b/toucan_connectors/google_spreadsheet/google_spreadsheet_connector.py @@ -26,15 +26,13 @@ class GoogleSpreadsheetDataSource(ToucanDataSource): ) -class GoogleSpreadsheetConnector(ToucanConnector): +class GoogleSpreadsheetConnector(ToucanConnector, data_source_model=GoogleSpreadsheetDataSource): """ For authentication, download an authentication file from console.developper.com and use the values here. This is an oauth2 credential file. For more information see this: http://gspread.readthedocs.io/en/latest/oauth2.html """ - data_source_model: GoogleSpreadsheetDataSource - credentials: GoogleCredentials = Field( ..., title='Google Credentials', diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index 88ced0c25..557a1098e 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -124,8 +124,7 @@ def model_json_schema( return schema -class HttpAPIConnector(ToucanConnector): - data_source_model: HttpAPIDataSource +class HttpAPIConnector(ToucanConnector, data_source_model=HttpAPIDataSource): responsetype: ResponseType = Field(ResponseType.json, title='Content-type of response') baseroute: AnyHttpUrl = Field(..., title='Baseroute URL', description='Baseroute URL') cert: List[FilePath] = Field( diff --git a/toucan_connectors/hubspot/hubspot_connector.py b/toucan_connectors/hubspot/hubspot_connector.py index 53c2f5f2f..42add2ce8 100644 --- a/toucan_connectors/hubspot/hubspot_connector.py +++ b/toucan_connectors/hubspot/hubspot_connector.py @@ -68,12 +68,11 @@ class HubspotDataSource(ToucanDataSource): parameters: Dict = Field(None) -class HubspotConnector(ToucanConnector): +class HubspotConnector(ToucanConnector, data_source_model=HubspotDataSource): _auth_flow = 'oauth2' auth_flow_id: Optional[str] = None _oauth_trigger = 'instance' oauth2_version: str = Field('1', **{'ui.hidden': True}) - data_source_model: HubspotDataSource _oauth2_connector: OAuth2Connector = PrivateAttr() def __init__(self, **kwargs) -> None: diff --git a/toucan_connectors/hubspot_private_app/hubspot_connector.py b/toucan_connectors/hubspot_private_app/hubspot_connector.py index 694afafce..a4dcead7a 100644 --- a/toucan_connectors/hubspot_private_app/hubspot_connector.py +++ b/toucan_connectors/hubspot_private_app/hubspot_connector.py @@ -91,8 +91,7 @@ def _page_api_for(api: _Api, dataset: HubspotDataset) -> _PageApi: return api.basic_api -class HubspotConnector(ToucanConnector): - data_source_model: type[HubspotDataSource] = HubspotDataSource +class HubspotConnector(ToucanConnector, data_source_model=HubspotDataSource): access_token: PlainJsonSecretStr = Field( ..., description='An API key for the target private app' ) diff --git a/toucan_connectors/linkedinads/linkedinads_connector.py b/toucan_connectors/linkedinads/linkedinads_connector.py index 77bb71edd..19c61fbb3 100644 --- a/toucan_connectors/linkedinads/linkedinads_connector.py +++ b/toucan_connectors/linkedinads/linkedinads_connector.py @@ -102,10 +102,9 @@ def model_json_schema( return schema -class LinkedinadsConnector(ToucanConnector): +class LinkedinadsConnector(ToucanConnector, data_source_model=LinkedinadsDataSource): """The LinkedinAds connector.""" - data_source_model: LinkedinadsDataSource _auth_flow = 'oauth2' auth_flow_id: Optional[ str diff --git a/toucan_connectors/micro_strategy/micro_strategy_connector.py b/toucan_connectors/micro_strategy/micro_strategy_connector.py index b8881dc54..9761d11a4 100644 --- a/toucan_connectors/micro_strategy/micro_strategy_connector.py +++ b/toucan_connectors/micro_strategy/micro_strategy_connector.py @@ -54,14 +54,12 @@ class MicroStrategyDataSource(ToucanDataSource): ) -class MicroStrategyConnector(ToucanConnector): +class MicroStrategyConnector(ToucanConnector, data_source_model=MicroStrategyDataSource): """ Import data from MicroStrategy using the [JSON Data API](http://bit.ly/2HCzf04) for cubes and reports. """ - data_source_model: MicroStrategyDataSource - base_url: HttpUrl = Field( ..., title='API base URL', @@ -82,7 +80,7 @@ class MicroStrategyConnector(ToucanConnector): def _retrieve_metadata(self, data_source: MicroStrategyDataSource) -> pd.DataFrame: client = Client( - self.base_url, self.project_id, self.username, self.password.get_secret_value() + str(self.base_url), self.project_id, self.username, self.password.get_secret_value() ) results = client.list_objects( @@ -100,7 +98,7 @@ def _retrieve_data(self, data_source: MicroStrategyDataSource) -> pd.DataFrame: if not self.password: self.password = PlainJsonSecretStr('') client = Client( - self.base_url, self.project_id, self.username, self.password.get_secret_value() + str(self.base_url), self.project_id, self.username, self.password.get_secret_value() ) query_func = getattr(client, data_source.dataset) diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 9aacf2fdc..61067207a 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -139,11 +139,11 @@ def get_form(cls, connector: 'MongoConnector', current_config): return create_model('FormSchema', **constraints, __base__=cls).schema() -class MongoConnector(ToucanConnector, VersionableEngineConnector): +class MongoConnector( + ToucanConnector, VersionableEngineConnector, data_source_model=MongoDataSource +): """Retrieve data from a [MongoDB](https://www.mongodb.com/) database.""" - data_source_model: MongoDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/mssql/mssql_connector.py b/toucan_connectors/mssql/mssql_connector.py index eb26aab47..cd13b5b74 100644 --- a/toucan_connectors/mssql/mssql_connector.py +++ b/toucan_connectors/mssql/mssql_connector.py @@ -74,13 +74,11 @@ def get_form(cls, connector: 'MSSQLConnector', current_config): return create_model('FormSchema', **constraints, __base__=cls).schema() -class MSSQLConnector(ToucanConnector): +class MSSQLConnector(ToucanConnector, data_source_model=MSSQLDataSource): """ Import data from Microsoft SQL Server. """ - data_source_model: MSSQLDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py index e63987abd..6edff9df8 100644 --- a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py +++ b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py @@ -75,13 +75,11 @@ def get_form(cls, connector: 'MSSQLConnector', current_config): return create_model('FormSchema', **constraints, __base__=cls).schema() -class MSSQLConnector(ToucanConnector): +class MSSQLConnector(ToucanConnector, data_source_model=MSSQLDataSource): """ Import data from Microsoft SQL Server. """ - data_source_model: MSSQLDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index 1e9daa8ae..5f8e322c0 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -96,13 +96,16 @@ class SSLMode(str, Enum): REQUIRED = 'REQUIRED' -class MySQLConnector(ToucanConnector, DiscoverableConnector, VersionableEngineConnector): +class MySQLConnector( + ToucanConnector, + DiscoverableConnector, + VersionableEngineConnector, + data_source_model=MySQLDataSource, +): """ Import data from MySQL database. """ - data_source_model: MySQLDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/net_explorer/net_explorer_connector.py b/toucan_connectors/net_explorer/net_explorer_connector.py index 949526fd6..a4e76cd15 100755 --- a/toucan_connectors/net_explorer/net_explorer_connector.py +++ b/toucan_connectors/net_explorer/net_explorer_connector.py @@ -16,13 +16,8 @@ class NetExplorerDataSource(ToucanDataSource): sheet: Optional[str] = 0 -class NetExplorerConnector(ToucanConnector): - data_source_model: NetExplorerDataSource - instance_url: str = Field( - None, - Title='Instance URL', - placeholder='exemple.netexplorer.pro', - ) +class NetExplorerConnector(ToucanConnector, data_source_model=NetExplorerDataSource): + instance_url: str = Field(None, Title='Instance URL', placeholder='exemple.netexplorer.pro') user: str password: PlainJsonSecretStr diff --git a/toucan_connectors/odata/odata_connector.py b/toucan_connectors/odata/odata_connector.py index 6451a0d5a..53006b110 100644 --- a/toucan_connectors/odata/odata_connector.py +++ b/toucan_connectors/odata/odata_connector.py @@ -36,9 +36,7 @@ class ODataDataSource(ToucanDataSource): ) -class ODataConnector(ToucanConnector): - data_source_model: ODataDataSource - +class ODataConnector(ToucanConnector, data_source_model=ODataDataSource): baseroute: HttpUrl = Field(..., title='API endpoint', description='Baseroute URL') auth: Auth = Field(None, title='Authentication type') diff --git a/toucan_connectors/odbc/odbc_connector.py b/toucan_connectors/odbc/odbc_connector.py index 664478ebf..0a4424652 100644 --- a/toucan_connectors/odbc/odbc_connector.py +++ b/toucan_connectors/odbc/odbc_connector.py @@ -13,13 +13,11 @@ class OdbcDataSource(ToucanDataSource): ) -class OdbcConnector(ToucanConnector): +class OdbcConnector(ToucanConnector, data_source_model=OdbcDataSource): """ Import data through ODBC apis """ - data_source_model: OdbcDataSource - connection_string: str = Field(..., description='The connection string') autocommit: bool = False ansi: bool = False diff --git a/toucan_connectors/one_drive/one_drive_connector.py b/toucan_connectors/one_drive/one_drive_connector.py index 7b3efdb2b..d39ee5497 100755 --- a/toucan_connectors/one_drive/one_drive_connector.py +++ b/toucan_connectors/one_drive/one_drive_connector.py @@ -71,9 +71,7 @@ def _prepare_workbook_elements(data_source): return workbook_elements_list, workbook_key_column -class OneDriveConnector(ToucanConnector): - data_source_model: OneDriveDataSource - +class OneDriveConnector(ToucanConnector, data_source_model=OneDriveDataSource): _auth_flow = 'oauth2' _oauth_trigger = 'connector' oauth2_version: str = Field('1', **{'ui.hidden': True}) diff --git a/toucan_connectors/oracle_sql/oracle_sql_connector.py b/toucan_connectors/oracle_sql/oracle_sql_connector.py index 03efd1c71..b3bf199d4 100644 --- a/toucan_connectors/oracle_sql/oracle_sql_connector.py +++ b/toucan_connectors/oracle_sql/oracle_sql_connector.py @@ -56,9 +56,7 @@ def get_form(cls, connector: 'OracleSQLConnector', current_config): return create_model('FormSchema', **constraints, __base__=cls).schema() -class OracleSQLConnector(ToucanConnector): - data_source_model: OracleSQLDataSource - +class OracleSQLConnector(ToucanConnector, data_source_model=OracleSQLDataSource): dsn: str = Field( ..., description='A path following the ' diff --git a/toucan_connectors/peakina/peakina_connector.py b/toucan_connectors/peakina/peakina_connector.py index 8cd4a2d5c..db812cc35 100644 --- a/toucan_connectors/peakina/peakina_connector.py +++ b/toucan_connectors/peakina/peakina_connector.py @@ -14,8 +14,6 @@ def __init__(self, **data: Any) -> None: super().__init__(**data) -class PeakinaConnector(ToucanConnector): - data_source_model: type[PeakinaDataSource] = PeakinaDataSource - +class PeakinaConnector(ToucanConnector, data_source_model=PeakinaDataSource): def _retrieve_data(self, data_source: PeakinaDataSource) -> pd.DataFrame: return data_source.get_df() diff --git a/toucan_connectors/postgres/postgresql_connector.py b/toucan_connectors/postgres/postgresql_connector.py index 6768328ea..b7a77e7e5 100644 --- a/toucan_connectors/postgres/postgresql_connector.py +++ b/toucan_connectors/postgres/postgresql_connector.py @@ -89,13 +89,16 @@ def get_form(cls, connector: 'PostgresConnector', current_config): return create_model('FormSchema', **constraints, __base__=cls).schema() -class PostgresConnector(ToucanConnector, DiscoverableConnector, VersionableEngineConnector): +class PostgresConnector( + ToucanConnector, + DiscoverableConnector, + VersionableEngineConnector, + data_source_model=PostgresDataSource, +): """ Import data from PostgreSQL. """ - data_source_model: PostgresDataSource - host: str = Field( None, description='The listening address of your database server (IP adress or hostname)' ) diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index c94f19c99..9caa05363 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -106,8 +106,9 @@ def get_form(cls, connector: 'RedshiftConnector', current_config: dict[str, Any] ).schema() -class RedshiftConnector(ToucanConnector, DiscoverableConnector): - data_source_model: RedshiftDataSource +class RedshiftConnector( + ToucanConnector, DiscoverableConnector, data_source_model=RedshiftDataSource +): authentication_method: AuthenticationMethod = Field( None, title='Authentication Method', diff --git a/toucan_connectors/salesforce/salesforce_connector.py b/toucan_connectors/salesforce/salesforce_connector.py index 96f18aba5..a86c8b03c 100644 --- a/toucan_connectors/salesforce/salesforce_connector.py +++ b/toucan_connectors/salesforce/salesforce_connector.py @@ -49,10 +49,9 @@ class SalesforceDataSource(ToucanDataSource): ) -class SalesforceConnector(ToucanConnector): +class SalesforceConnector(ToucanConnector, data_source_model=SalesforceDataSource): _auth_flow = 'oauth2' auth_flow_id: Optional[str] = None - data_source_model: SalesforceDataSource instance_url: str = Field( None, title='instance url', diff --git a/toucan_connectors/sap_hana/sap_hana_connector.py b/toucan_connectors/sap_hana/sap_hana_connector.py index e51c38e52..2055461b0 100644 --- a/toucan_connectors/sap_hana/sap_hana_connector.py +++ b/toucan_connectors/sap_hana/sap_hana_connector.py @@ -12,13 +12,11 @@ class SapHanaDataSource(ToucanDataSource): ) -class SapHanaConnector(ToucanConnector): +class SapHanaConnector(ToucanConnector, data_source_model=SapHanaDataSource): """ Import data from Sap Hana. """ - data_source_model: SapHanaDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 495af2d78..cb41b1ce0 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -124,7 +124,11 @@ def _snowflake_connection( ) -class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConnector): +class SnowflakeConnector( + ToucanConnector, + DiscoverableConnector, + data_source_model=SnowflakeDataSource, +): """ Import data from Snowflake data warehouse. """ @@ -134,8 +138,6 @@ class SnowflakeConnector(ToucanConnector[SnowflakeDataSource], DiscoverableConne sso_credentials_keeper: Any = None user_tokens_keeper: Any = None - data_source_model: type[SnowflakeDataSource] = SnowflakeDataSource - account: str = Field( ..., description='The full name of your Snowflake account. ' diff --git a/toucan_connectors/snowflake_common.py b/toucan_connectors/snowflake_common.py index cca30b936..cbc40bf71 100644 --- a/toucan_connectors/snowflake_common.py +++ b/toucan_connectors/snowflake_common.py @@ -42,7 +42,7 @@ class SnowflakeConnectorWarehouseDoesNotExists(Exception): class SfDataSource(ToucanDataSource): database: str = Field(..., description='The name of the database you want to query') - warehouse: str = Field(None, description='The name of the warehouse you want to query') + warehouse: str | None = Field(None, description='The name of the warehouse you want to query') query: Annotated[str, StringConstraints(min_length=1)] = Field( ..., description='You can write your SQL query here', widget='sql' diff --git a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py index 1e510e4b7..d6329141b 100644 --- a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py +++ b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py @@ -61,7 +61,7 @@ def get_form(cls, connector: 'SnowflakeoAuth2Connector', current_config): return res -class SnowflakeoAuth2Connector(ToucanConnector): +class SnowflakeoAuth2Connector(ToucanConnector, data_source_model=SnowflakeoAuth2DataSource): client_id: str = Field( '', title='Client ID', @@ -100,7 +100,6 @@ class SnowflakeoAuth2Connector(ToucanConnector): 'in the form of: "your_account_name.region_id.cloud_platform". See more details ' 'here.', ) - data_source_model: SnowflakeoAuth2DataSource default_warehouse: str = Field( ..., description='The default warehouse that shall be used for any data source' ) diff --git a/toucan_connectors/soap/soap_connector.py b/toucan_connectors/soap/soap_connector.py index a795c7afb..e8be047bf 100644 --- a/toucan_connectors/soap/soap_connector.py +++ b/toucan_connectors/soap/soap_connector.py @@ -64,8 +64,7 @@ def get_form(cls, connector: 'SoapConnector', current_config): return res -class SoapConnector(ToucanConnector): - data_source_model: SoapDataSource +class SoapConnector(ToucanConnector, data_source_model=SoapDataSource): headers: dict = Field( None, description='JSON object of HTTP headers to send with every HTTP request', diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index 617720377..928bc513b 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -8,7 +8,17 @@ from abc import ABC, ABCMeta, abstractmethod from enum import Enum from functools import reduce, wraps -from typing import Annotated, Any, Generic, Iterable, NamedTuple, Type, TypeVar, Union +from typing import ( + TYPE_CHECKING, + Annotated, + Any, + Generic, + Iterable, + NamedTuple, + Type, + TypeVar, + Union, +) import pandas as pd import tenacity as tny @@ -29,6 +39,10 @@ except ImportError: pass +if TYPE_CHECKING: + from pydantic.main import IncEx + + LOGGER = logging.getLogger(__name__) @@ -283,11 +297,15 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): the `_retry_on` class attribute in your concrete connector class. """ - data_source_model: type[DS] + @classmethod + def __init_subclass__(cls, /, *, data_source_model: type[DS]): + cls.logger = logging.getLogger(cls.__name__) # type:ignore[attr-defined] + cls.data_source_model = data_source_model # type:ignore[attr-defined] + name: str = Field(...) retry_policy: RetryPolicy | None = RetryPolicy() _retry_on: Iterable[Type[BaseException]] = () - type: str + type: str | None = None secrets_storage_version: str = Field('1', **_UI_HIDDEN) # type:ignore[pydantic-field] # Default ttl for all connector's queries (overridable at the data_source level) @@ -302,10 +320,6 @@ class ToucanConnector(BaseModel, Generic[DS], metaclass=ABCMeta): identifier: str = Field(None, **_UI_HIDDEN) # type:ignore[pydantic-field] model_config = ConfigDict(extra='forbid', validate_assignment=True) - @classmethod - def __init_subclass__(cls): - cls.logger = logging.getLogger(cls.__name__) - def bearer_oauth_get_endpoint( self, endpoint: str, @@ -425,7 +439,7 @@ def get_unique_identifier(self) -> str: Used by `get_cache_key` method. """ - return self.json() + return self.model_dump_json() def _get_unique_datasource_identifier(self, data_source: DS) -> dict: # By default we don't know which variable syntax is be supported by the inheriting connector, @@ -472,6 +486,33 @@ def get_identifier(self): def describe(self, data_source: DS): """ """ + # def model_dump_json( + # self, + # *, + # indent: int | None = None, + # include: 'IncEx' = None, + # exclude: 'IncEx' = None, + # by_alias: bool = False, + # exclude_unset: bool = False, + # exclude_defaults: bool = False, + # exclude_none: bool = False, + # round_trip: bool = False, + # warnings: bool = True, + # ) -> str: + # if exclude is None: + # exclude = {'data_source_model'} + # return super().model_dump_json( + # indent=indent, + # include=include, + # exclude=exclude, + # by_alias=by_alias, + # exclude_unset=exclude_unset, + # exclude_defaults=exclude_defaults, + # exclude_none=exclude_none, + # round_trip=round_trip, + # warnings=warnings, + # ) + TableInfo = dict[str, Union[str, list[dict[str, str]]]] diff --git a/toucan_connectors/toucan_toco/toucan_toco_connector.py b/toucan_connectors/toucan_toco/toucan_toco_connector.py index 5b3ab4583..553623b9d 100644 --- a/toucan_connectors/toucan_toco/toucan_toco_connector.py +++ b/toucan_connectors/toucan_toco/toucan_toco_connector.py @@ -21,13 +21,11 @@ class ToucanTocoDataSource(ToucanDataSource): all_small_apps: bool = False -class ToucanTocoConnector(ToucanConnector): +class ToucanTocoConnector(ToucanConnector, data_source_model=ToucanTocoDataSource): """ Get data from a Toucan Toco instance, usefull to build analytics applications. """ - data_source_model: ToucanTocoDataSource - host: str username: str password: str diff --git a/toucan_connectors/trello/trello_connector.py b/toucan_connectors/trello/trello_connector.py index b62cced09..70dc9ff5f 100644 --- a/toucan_connectors/trello/trello_connector.py +++ b/toucan_connectors/trello/trello_connector.py @@ -65,11 +65,9 @@ class TrelloDataSource(ToucanDataSource): filter: CardsFilter = CardsFilter.open -class TrelloConnector(ToucanConnector): - data_source_model: TrelloDataSource - - key_id: str = None - token: str = None +class TrelloConnector(ToucanConnector, data_source_model=TrelloDataSource): + key_id: str | None = None + token: str | None = None def get_board(self, path, **custom_params): return requests.get( diff --git a/toucan_connectors/wootric/wootric_connector.py b/toucan_connectors/wootric/wootric_connector.py index 0aebd72ee..1f0d738eb 100644 --- a/toucan_connectors/wootric/wootric_connector.py +++ b/toucan_connectors/wootric/wootric_connector.py @@ -122,9 +122,7 @@ class WootricDataSource(ToucanDataSource): ) -class WootricConnector(ToucanConnector): - data_source_model: WootricDataSource - +class WootricConnector(ToucanConnector, data_source_model=WootricDataSource): client_id: str client_secret: str api_version: str = 'v1' From 0066455f39e1dfe5ea484ff55fdf66eed66be14d Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Sep 2023 16:55:43 +0200 Subject: [PATCH 12/27] update tests Signed-off-by: Luka Peschke --- tests/redshift/test_redshift.py | 39 ++++----------- tests/soap/test_soap.py | 2 +- tests/test_connector.py | 88 +++++++++++++-------------------- 3 files changed, 45 insertions(+), 84 deletions(-) diff --git a/tests/redshift/test_redshift.py b/tests/redshift/test_redshift.py index 4f112ecc5..e7be5bae1 100644 --- a/tests/redshift/test_redshift.py +++ b/tests/redshift/test_redshift.py @@ -1,6 +1,7 @@ from unittest.mock import Mock, patch import pytest +from pytest_mock import MockerFixture from redshift_connector.error import InterfaceError, OperationalError, ProgrammingError from toucan_connectors.pagination import ( @@ -78,33 +79,10 @@ def redshift_datasource(): ) -def test_config_schema_extra(): - schema = { - 'properties': { - 'type': 'type_test', - 'name': 'name_test', - 'host': 'host_test', - 'port': 0, - 'cluster_identifier': 'cluster_identifier_test', - 'db_user': 'db_user_test', - 'connect_timeout': 'connect_timeout_test', - 'authentication_method': 'authentication_method_test', - 'user': 'user_test', - 'password': 'password_test', - 'default_database': 'dev', - 'access_key_id': 'access_key_id_test', - 'secret_access_key': 'secret_access_key_test', - 'session_token': 'session_token_test', - 'profile': 'profile_test', - 'region': 'region_test', - 'enable_tcp_keepalive': True, - } - } - RedshiftConnector.Config().schema_extra(schema) - assert schema['properties'] is not None - keys = list(schema['properties'].keys()) - for i in range(len(keys)): - assert keys[i] == ORDERED_KEYS[i] +def test_model_json_schema(): + schema = RedshiftConnector.model_json_schema() + for key, expected_key in zip(schema['properties'].keys(), ORDERED_KEYS): + assert key == expected_key def test_redshiftdatasource_init_(redshift_datasource): @@ -113,10 +91,11 @@ def test_redshiftdatasource_init_(redshift_datasource): assert hasattr(ds, 'query_object') -@patch.object(RedshiftConnector, '_retrieve_tables') -def test_redshiftdatasource_get_form(redshift_connector, redshift_datasource): +def test_redshiftdatasource_get_form( + redshift_connector, redshift_datasource, mocker: MockerFixture +): current_config = {'database': 'dev'} - redshift_connector._retrieve_tables.return_value = ['table1', 'table2', 'table3'] + mocker.patch.object(RedshiftConnector, 'available_dbs', new=['one', 'two']) result = redshift_datasource.get_form(redshift_connector, current_config) assert result['properties']['parameters']['title'] == 'Parameters' assert result['properties']['domain']['title'] == 'Domain' diff --git a/tests/soap/test_soap.py b/tests/soap/test_soap.py index 90774d861..0165e921c 100644 --- a/tests/soap/test_soap.py +++ b/tests/soap/test_soap.py @@ -65,7 +65,7 @@ def test_get_form(mocker, connector, create_datasource): ) form = create_datasource.get_form(connector, {}) assert form['properties']['parameters']['description'] == 'Services documentation:
coucou' - assert form['definitions']['method']['enum'][0] == 'fake_func' + assert form['$defs']['method']['const'] == 'fake_func' def test_create_client(mocker, connector): diff --git a/tests/test_connector.py b/tests/test_connector.py index 9f3730362..cfebcebe3 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -4,7 +4,7 @@ import pandas as pd import pytest import tenacity as tny -from pydantic import create_model +from pydantic import ValidationError, create_model from toucan_connectors.common import ConnectorStatus from toucan_connectors.google_sheets_2.google_sheets_2_connector import GoogleSheets2Connector @@ -29,9 +29,8 @@ class DataSource(ToucanDataSource): parameters: dict = {} -class DataConnector(ToucanConnector): - type = 'MyDB' - data_source_model: DataSource +class DataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyDB' a_parameter: str = '' def _retrieve_data(self, data_source): @@ -41,21 +40,18 @@ def _retrieve_data(self, data_source): ################################################ def test_missing_attributes(): # missing data_source_model - with pytest.raises(TypeError) as exc_info: + with pytest.raises(TypeError, match='data_source_model'): class MissingDataConnector2(ToucanConnector): - type = 'MyDB' + type: str = 'MyDB' def _retrieve_data(self, data_source): pass - assert str(exc_info.value) == "MissingDataConnector2 has no 'data_source_model' attribute." - def test_no_get_df(): - class BadDataConnector(ToucanConnector): - type = 'MyDB' - data_source_model = 'asd' + class BadDataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyDB' with pytest.raises(TypeError): BadDataConnector(name='my_name') @@ -74,9 +70,8 @@ def test_validate(): def test_formated_engine_version(): - class DataConnector(ToucanConnector, VersionableEngineConnector): - type = 'MyDB' - data_source_model: DataSource + class DataConnector(ToucanConnector, VersionableEngineConnector, data_source_model=DataSource): + type: str = 'MyDB' def get_engine_version(self) -> tuple: return super().get_engine_version() @@ -99,9 +94,8 @@ def _retrieve_data(self, datasource): def test_get_df_with_permissions(): - class DataConnector(ToucanConnector): - type = 'MyDB' - data_source_model: DataSource + class DataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyDB' def _retrieve_data(self, datasource): return pd.DataFrame({'A': [1, 2]}) @@ -113,9 +107,8 @@ def _retrieve_data(self, datasource): def test_get_slice(): - class DataConnector(ToucanConnector): - type = 'MyDB' - data_source_model = 'asd' + class DataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyDB' def _retrieve_data(self, datasource): return pd.DataFrame({'A': [1, 2, 3, 4, 5]}) @@ -144,9 +137,8 @@ def _retrieve_data(self, datasource): def test_explain(): - class DataConnector(ToucanConnector): - type = 'MyDB' - data_source_model = 'asd' + class DataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyDB' def _retrieve_data(self, datasource): return pd.DataFrame() @@ -166,7 +158,7 @@ def test_get_cache_key(): key = connector.get_cache_key(ds) # We should get a deterministic identifier: # /!\ the identifier will change if the model of the connector or the datasource changes - assert key == '10ae8360-5e30-319e-8cb8-a8a817826d12' + assert key == 'cc66ddcd-e717-381f-838e-f960e6cb410e' ds.query = 'wow' key2 = connector.get_cache_key(ds) @@ -212,9 +204,8 @@ def test_get_cache_key_should_be_different_with_different_permissions(): assert key_a1 != key_a2 -class UnreliableDataConnector(ToucanConnector): - type = 'MyUnreliableDB' - data_source_model: DataSource +class UnreliableDataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyUnreliableDB' def _retrieve_data(self, data_source, logbook=[]): if len(logbook) < 3: @@ -231,9 +222,8 @@ def test_max_attempt_df(): assert result == 42 -class CustomPolicyDataConnector(ToucanConnector): - type = 'MyUnreliableDB' - data_source_model: DataSource +class CustomPolicyDataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyUnreliableDB' def _retrieve_data(self, data_source, logbook=[]): if len(logbook) < 3: @@ -253,9 +243,8 @@ def test_custom_max_attempt_df(): assert result['1'].values.tolist() == [42, 32] -class CustomRetryOnDataConnector(ToucanConnector): - type = 'MyUnreliableDB' - data_source_model: DataSource +class CustomRetryOnDataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyUnreliableDB' _retry_on = (ValueError,) def _retrieve_data(self, data_source, logbook=[]): @@ -272,9 +261,8 @@ def test_custom_retry_on_df(): udc.get_df({}) -class CustomNoRetryOnDataConnector(ToucanConnector): - type = 'MyUnreliableDB' - data_source_model: DataSource +class CustomNoRetryOnDataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyUnreliableDB' @property def retry_decorator(self): @@ -297,18 +285,17 @@ def test_no_retry_on_df(): def test_strlist_to_enum_required(): """It should be required by default""" model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'])) - assert model.schema() == { + assert model.model_json_schema() == { 'title': 'Test', 'type': 'object', - 'definitions': { + '$defs': { 'pokemon': { - 'description': 'An enumeration.', 'enum': ['pika', 'bulbi'], 'title': 'pokemon', 'type': 'string', } }, - 'properties': {'pokemon': {'$ref': '#/definitions/pokemon'}}, + 'properties': {'pokemon': {'$ref': '#/$defs/pokemon'}}, 'required': ['pokemon'], } @@ -316,27 +303,24 @@ def test_strlist_to_enum_required(): def test_strlist_to_enum_default_value(): """It should be possible to add a default value (not required)""" model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'], 'pika')) - assert model.schema() == { + assert model.model_json_schema() == { 'title': 'Test', 'type': 'object', - 'definitions': { + '$defs': { 'pokemon': { - 'description': 'An enumeration.', 'enum': ['pika', 'bulbi'], 'title': 'pokemon', 'type': 'string', } }, - 'properties': { - 'pokemon': {'allOf': [{'$ref': '#/definitions/pokemon'}], 'default': 'pika'} - }, + 'properties': {'pokemon': {'allOf': [{'$ref': '#/$defs/pokemon'}], 'default': 'pika'}}, } def test_should_return_connector_config_form(): assert ( get_connector_secrets_form(GoogleSheets2Connector).secrets_schema - == OAuth2ConnectorConfig.schema() + == OAuth2ConnectorConfig.model_json_schema() ) assert get_connector_secrets_form(MongoConnector) is None @@ -344,9 +328,8 @@ def test_should_return_connector_config_form(): def test_get_df_int_column(mocker): """The int column should be casted as str""" - class DataConnector(ToucanConnector): - type = 'MyDB' - data_source_model: DataSource + class DataConnector(ToucanConnector, data_source_model=DataSource): + type: str = 'MyDB' def _retrieve_data(self, datasource): return pd.DataFrame({0: [1, 2]}) @@ -356,9 +339,8 @@ def _retrieve_data(self, datasource): def test_default_implementation_of_discoverable_connector(): - class DataConnector(ToucanConnector, DiscoverableConnector): - type = 'MyDB' - data_source_model: DataSource + class DataConnector(ToucanConnector, DiscoverableConnector, data_source_model=DataSource): + type: str = 'MyDB' def _retrieve_data(self, datasource): return pd.DataFrame() From 9f0ce8798531d5a26fdace0cf98290d03c710c47 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 12 Sep 2023 15:26:07 +0200 Subject: [PATCH 13/27] fix tests Signed-off-by: Luka Peschke --- tests/anaplan/test_anaplan.py | 8 ++- tests/awsathena/test_awsathena.py | 2 +- tests/clickhouse/test_clickhouse.py | 42 ++++---------- tests/github/test_github.py | 2 +- tests/google_adwords/test_google_adwords.py | 44 ++++++-------- .../google_big_query/test_google_big_query.py | 5 +- tests/google_sheets/test_google_sheets.py | 2 +- tests/google_sheets_2/test_google_sheets_2.py | 13 +++-- tests/http_api/test_http_api.py | 37 ++++-------- tests/linkedinads/test_linkedinads.py | 37 ++++-------- tests/mongo/test_mongo.py | 26 +++++---- tests/mysql/test_mysql.py | 10 ++-- tests/postgres/test_postgres.py | 15 ++--- tests/snowflake/test_snowflake.py | 4 +- tests/test_connector.py | 2 +- tests/test_datasource.py | 45 ++++++++------- .../google_adwords_connector.py | 2 +- .../google_analytics_connector.py | 54 +++++++++--------- .../google_sheets_2_connector.py | 14 ++--- .../http_api/http_api_connector.py | 2 +- .../linkedinads/linkedinads_connector.py | 6 +- toucan_connectors/mongo/mongo_connector.py | 9 ++- toucan_connectors/mysql/mysql_connector.py | 6 +- toucan_connectors/odata/odata_connector.py | 4 +- toucan_connectors/toucan_connector.py | 57 ++++--------------- 25 files changed, 178 insertions(+), 270 deletions(-) diff --git a/tests/anaplan/test_anaplan.py b/tests/anaplan/test_anaplan.py index 5815c4027..a0fcb5ddf 100644 --- a/tests/anaplan/test_anaplan.py +++ b/tests/anaplan/test_anaplan.py @@ -140,9 +140,11 @@ def test_get_form(connector): # Ensure we've only requested a token once responses.assert_call_count('https://auth.anaplan.com/token/authenticate', 1) - assert form_schema['definitions']['workspace_id']['enum'] == ['w1 - Workspace One'] - assert form_schema['definitions']['model_id']['enum'] == ['m1 - Model One'] - assert form_schema['definitions']['view_id']['enum'] == ['m1v1 - View One', 'm1v2 - View Two'] + # We have a single values for these, so it's a const + assert form_schema['$defs']['workspace_id']['const'] == 'w1 - Workspace One' + assert form_schema['$defs']['model_id']['const'] == 'm1 - Model One' + # We have several values for this one, so enum + assert form_schema['$defs']['view_id']['enum'] == ['m1v1 - View One', 'm1v2 - View Two'] @responses.activate diff --git a/tests/awsathena/test_awsathena.py b/tests/awsathena/test_awsathena.py index 66bf59566..49f18a927 100644 --- a/tests/awsathena/test_awsathena.py +++ b/tests/awsathena/test_awsathena.py @@ -193,7 +193,7 @@ def test_athenadatasource_get_form( assert result['properties']['domain']['title'] == 'Domain' assert result['properties']['validation']['title'] == 'Validation' assert result['required'] == ['domain', 'name', 'database'] - assert result['definitions']['database']['enum'] == ['db1', 'db2'] + assert result['$defs']['database']['enum'] == ['db1', 'db2'] @pytest.mark.usefixtures('data_source', 'mocked_boto_session') diff --git a/tests/clickhouse/test_clickhouse.py b/tests/clickhouse/test_clickhouse.py index 9c714ee3f..26dae280c 100644 --- a/tests/clickhouse/test_clickhouse.py +++ b/tests/clickhouse/test_clickhouse.py @@ -151,20 +151,14 @@ def test_get_form_query_with_good_database(clickhouse_connector): """It should give suggestions of the collections""" current_config = {'database': 'clickhouse_db'} form = ClickhouseDataSource.get_form(clickhouse_connector, current_config) - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'enum': ['INFORMATION_SCHEMA', 'clickhouse_db', 'default', 'information_schema'], 'type': 'string', } - assert form['properties']['table'] == {'$ref': '#/definitions/table'} - assert form['definitions']['table'] == { - 'title': 'table', - 'description': 'An enumeration.', - 'type': 'string', - 'enum': ['city'], - } + assert form['properties']['table'] == {'allOf': [{'$ref': '#/$defs/table'}], 'default': None} + assert form['$defs']['table'] == {'const': 'city', 'title': 'table', 'type': 'string'} assert form['required'] == ['domain', 'name', 'database'] @@ -175,7 +169,7 @@ def test_get_form_connection_fails(mocker, clickhouse_connector): assert 'table' in form['properties'] -def test_schema_extra(): +def test_model_json_schema(): data_source_spec = { 'domain': 'Clickhouse test', 'type': 'external_database', @@ -184,25 +178,13 @@ def test_schema_extra(): 'query': 'SELECT * FROM city WHERE id in %(ids)s', 'parameters': {'ids': [3986, 3958]}, } - conf = ClickhouseDataSource(**data_source_spec).Config - schema = { - 'properties': { - 'query': 'bar', - 'parameters': 'bar', - 'table': 'bar', - 'database': 'bar', - } - } - conf.schema_extra(schema, model=ClickhouseDataSource) - - assert schema == { - 'properties': { - 'database': 'bar', - 'table': 'bar', - 'query': 'bar', - 'parameters': 'bar', - } - } + ds = ClickhouseDataSource(**data_source_spec) + assert list(ds.model_json_schema()['properties'].keys())[:4] == [ + 'database', + 'table', + 'query', + 'parameters', + ] def test_create_connections(): diff --git a/tests/github/test_github.py b/tests/github/test_github.py index 391765288..b755eb497 100644 --- a/tests/github/test_github.py +++ b/tests/github/test_github.py @@ -479,7 +479,7 @@ def test_datasource_get_form_no_secret(gc, remove_secrets): status=200, ) res = ds.get_form(connector=gc, current_config={}) - assert 'organization' not in res['definitions'].keys() + assert 'organization' not in res['$defs'].keys() def test_get_slice( diff --git a/tests/google_adwords/test_google_adwords.py b/tests/google_adwords/test_google_adwords.py index 0736eeb58..c7d34587a 100644 --- a/tests/google_adwords/test_google_adwords.py +++ b/tests/google_adwords/test_google_adwords.py @@ -3,6 +3,7 @@ import pandas as pd import responses from pytest import fixture +from pytest_mock import MockerFixture from toucan_connectors.common import HttpError from toucan_connectors.google_adwords.google_adwords_connector import ( @@ -126,37 +127,20 @@ def test_retrieve_tokens(mocker, connector): mock_oauth2_connector.retrieve_tokens.assert_called() -def test_schema_extra(build_data_service_source): +def test_model_json_schema(build_data_service_source: GoogleAdwordsDataSource): """ Check that schema_extra correctly structures the Data Source form """ - conf = build_data_service_source.Config - schema = { - 'properties': { - 'domain': 'bar', - 'service': 'foo', - 'columns': 'bababa', - 'from_clause': 'foofoo', - 'parameters': 'barbar', - 'during': 'foobar', - 'orderby': 'barfoo', - 'limit': 100, - } - } - conf.schema_extra(schema, model=GoogleAdwordsDataSource) - assert schema == { - 'properties': { - 'service': 'foo', - 'columns': 'bababa', - 'from_clause': 'foofoo', - 'parameters': 'barbar', - 'during': 'foobar', - 'orderby': 'barfoo', - 'limit': 100, - 'domain': 'bar', - } - } + assert list(build_data_service_source.model_json_schema()['properties'].keys())[:7] == [ + 'service', + 'columns', + 'from_clause', + 'parameters', + 'during', + 'orderby', + 'limit', + ] def test_get_connectors_secrets_form(connector): @@ -194,7 +178,11 @@ def test_authenticate_client(connector, mocker): mocked_refresh.assert_called_once() -def test_prepare_service_query(connector, mocker, build_data_service_source): +def test_prepare_service_query( + connector: GoogleAdwordsConnector, + mocker: MockerFixture, + build_data_service_source: GoogleAdwordsDataSource, +): """ Check that prepare_service_query is able to build & return a service and a built service query diff --git a/tests/google_big_query/test_google_big_query.py b/tests/google_big_query/test_google_big_query.py index 37845abe4..1ab67120c 100644 --- a/tests/google_big_query/test_google_big_query.py +++ b/tests/google_big_query/test_google_big_query.py @@ -663,12 +663,9 @@ def test_get_model_multi_location(mocker: MockFixture, _fixture_credentials) -> def test_get_form(mocker: MockFixture, _fixture_credentials: MockFixture) -> None: - def mock_available_schs(): - return ['ok', 'test'] - mocker.patch( 'toucan_connectors.google_big_query.google_big_query_connector.GoogleBigQueryConnector._available_schs', - return_value=mock_available_schs, + new=['ok', 'test'], ) assert ( diff --git a/tests/google_sheets/test_google_sheets.py b/tests/google_sheets/test_google_sheets.py index 9783cf786..fe4b59190 100644 --- a/tests/google_sheets/test_google_sheets.py +++ b/tests/google_sheets/test_google_sheets.py @@ -300,7 +300,7 @@ def test_get_form(mocker): current_config={'spreadsheet_id': 'test_spreadsheet_id'}, ) expected_results = ['sample data', 'animals'] - assert schema['definitions']['sheet']['enum'] == expected_results + assert schema['$defs']['sheet']['enum'] == expected_results def test_numeric_dateformat_(): diff --git a/tests/google_sheets_2/test_google_sheets_2.py b/tests/google_sheets_2/test_google_sheets_2.py index 73425a996..296b7515d 100644 --- a/tests/google_sheets_2/test_google_sheets_2.py +++ b/tests/google_sheets_2/test_google_sheets_2.py @@ -3,6 +3,7 @@ import pytest from pytest import fixture +from pytest_mock import MockerFixture from pytz import utc from toucan_connectors.common import HttpError @@ -82,8 +83,8 @@ async def test_authentified_fetch(mocker, con): def get_columns_in_schema(schema): """Pydantic generates schema slightly differently in python <=3.7 and in python 3.8""" try: - if schema.get('definitions'): - return schema['definitions']['sheet']['enum'] + if (defs := schema.get('definitions')) or (defs := schema.get('$defs')): + return defs['sheet']['enum'] else: return schema['properties']['sheet']['enum'] except KeyError: @@ -267,13 +268,15 @@ def test_get_slice(mocker, con, ds): assert ds.df.shape == (2, 2) -def test_get_slice_no_limit(mocker, con, ds): +def test_get_slice_no_limit( + mocker: MockerFixture, con: GoogleSheets2Connector, ds: GoogleSheets2DataSource +): """It should return a slice of spreadsheet""" mocker.patch.object(GoogleSheets2Connector, '_run_fetch', return_value=FAKE_SHEET) - ds = con.get_slice(ds, limit=None) + slice = con.get_slice(ds, limit=None) - assert ds.df.shape == (2, 2) + assert slice.df.shape == (2, 2) def test_schema_fields_order(con, ds): diff --git a/tests/http_api/test_http_api.py b/tests/http_api/test_http_api.py index 09e778cfb..2cb91d309 100644 --- a/tests/http_api/test_http_api.py +++ b/tests/http_api/test_http_api.py @@ -438,7 +438,7 @@ def test_oauth2_oidc_authentication(mocker): mock_session.assert_called_once() -def test_schema_extra(): +def test_model_json_schema(): data_source_spec = { 'data': '', 'domain': 'Clickhouse test', @@ -458,30 +458,15 @@ def test_schema_extra(): 'validation': {}, 'xpath': '', } - conf = HttpAPIDataSource(**data_source_spec).Config - - schema = { - 'properties': { - 'data': '', - 'proxies': {}, - 'filter': '', - 'flatten_column': '', - 'validation': {}, - 'xpath': '', - } - } - conf.schema_extra(schema, model=HttpAPIDataSource) - - assert schema == { - 'properties': { - 'proxies': {}, - 'flatten_column': '', - 'data': '', - 'xpath': '', - 'filter': '', - 'validation': {}, - } - } + ds = HttpAPIDataSource(**data_source_spec) + assert list(ds.model_json_schema()['properties'].keys())[-6:] == [ + 'proxies', + 'flatten_column', + 'data', + 'xpath', + 'filter', + 'validation', + ] def test_get_cache_key(connector, auth, data_source): @@ -489,7 +474,7 @@ def test_get_cache_key(connector, auth, data_source): data_source.parameters = {'first_name': 'raphael'} key = connector.get_cache_key(data_source) - assert key == 'fa95c942-9b94-3f07-9ed4-24c34abfbdae' + assert key == 'f24af0b5-f745-3961-8aec-a27d44543fb9' data_source.headers = {'name': '{{ first_name }}'} # change the templating style key2 = connector.get_cache_key(data_source) diff --git a/tests/linkedinads/test_linkedinads.py b/tests/linkedinads/test_linkedinads.py index 44a5b0d09..4ddb3edcc 100644 --- a/tests/linkedinads/test_linkedinads.py +++ b/tests/linkedinads/test_linkedinads.py @@ -106,7 +106,9 @@ def test__retrieve_data(connector, create_datasource): @responses.activate -def test__retrieve_data_no_nested_col(connector, create_datasource): +def test__retrieve_data_no_nested_col( + connector: LinkedinadsConnector, create_datasource: LinkedinadsDataSource +): create_datasource.flatten_column = None responses.add( method='GET', @@ -153,30 +155,15 @@ def test_retrieve_tokens(mocker, connector): mock_oauth2_connector.retrieve_tokens.assert_called() -def test_schema_extra(create_datasource): - conf = create_datasource.Config - schema = { - 'properties': { - 'time_granularity': 'bar', - 'flatten_column': 'bar', - 'parameters': 'bar', - 'finder_methods': 'bar', - 'start_date': 'bar', - 'end_date': 'bar', - } - } - conf.schema_extra(schema, model=LinkedinadsDataSource) - - assert schema == { - 'properties': { - 'finder_methods': 'bar', - 'start_date': 'bar', - 'end_date': 'bar', - 'time_granularity': 'bar', - 'flatten_column': 'bar', - 'parameters': 'bar', - } - } +def test_model_json_schema(create_datasource: LinkedinadsDataSource): + assert list(create_datasource.model_json_schema()['properties'].keys())[:6] == [ + 'finder_methods', + 'start_date', + 'end_date', + 'time_granularity', + 'flatten_column', + 'parameters', + ] @responses.activate diff --git a/tests/mongo/test_mongo.py b/tests/mongo/test_mongo.py index 570883d80..e81059176 100644 --- a/tests/mongo/test_mongo.py +++ b/tests/mongo/test_mongo.py @@ -1,6 +1,7 @@ import os import re from datetime import datetime +from typing import Callable import pandas as pd import pymongo @@ -239,7 +240,9 @@ def test_get_df_with_permissions(mongo_connector, mongo_datasource): assert df[['country', 'language', 'value', 'name']].equals(expected) -def test_get_slice(mongo_connector, mongo_datasource): +def test_get_slice( + mongo_connector: MongoConnector, mongo_datasource: Callable[..., MongoDataSource] +): datasource = mongo_datasource(collection='test_col', query={'domain': 'domain1'}) res = mongo_connector.get_slice(datasource) assert res.pagination_info.pagination_info.total_rows == 5 @@ -664,10 +667,9 @@ def test_get_form_empty_query(mongo_connector): current_config = {} form = MongoDataSource.get_form(mongo_connector, current_config) assert form['required'] == ['domain', 'name', 'database', 'collection'] - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'type': 'string', 'enum': ['admin', 'config', 'local', 'toucan'], } @@ -690,19 +692,17 @@ def test_get_form_query_with_good_database(mongo_connector): current_config = {'database': 'toucan'} form = MongoDataSource.get_form(mongo_connector, current_config) assert form['required'] == ['domain', 'name', 'database', 'collection'] - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'type': 'string', 'enum': ['admin', 'config', 'local', 'toucan'], } - assert form['properties']['collection'] == {'$ref': '#/definitions/collection'} - assert form['definitions']['collection'] == { + assert form['properties']['collection'] == {'$ref': '#/$defs/collection'} + assert form['$defs']['collection'] == { 'title': 'collection', - 'description': 'An enumeration.', 'type': 'string', - 'enum': ['test_col'], + 'const': 'test_col', } @@ -854,7 +854,9 @@ def test_get_cache_key(mongo_connector): assert conn1.get_cache_key() != conn2.get_cache_key() -def test_get_cache_key_with_datasource(mongo_connector, mongo_datasource): +def test_get_cache_key_with_datasource( + mongo_connector: MongoConnector, mongo_datasource: Callable[..., MongoConnector] +): datasource = mongo_datasource(collection='test_col', query={'domain': 'domain1'}) datasource_with_parameters = mongo_datasource( collection='test_col', query={'domain': '{{ DOMAIN }}'}, parameters={'DOMAIN': 'domain1'} diff --git a/tests/mysql/test_mysql.py b/tests/mysql/test_mysql.py index 95f63e701..2794921ec 100644 --- a/tests/mysql/test_mysql.py +++ b/tests/mysql/test_mysql.py @@ -290,10 +290,9 @@ def test_get_form_empty_query(mysql_connector): """It should give suggestions of the databases without changing the rest""" current_config = {} form = MySQLDataSource.get_form(mysql_connector, current_config) - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'enum': ['mysql_db', 'other_db'], 'type': 'string', } @@ -303,10 +302,9 @@ def test_get_form_query_with_good_database(mysql_connector): """It should give suggestions of the collections""" current_config = {'database': 'mysql_db'} form = MySQLDataSource.get_form(mysql_connector, current_config) - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'type': 'string', 'enum': ['mysql_db', 'other_db'], } diff --git a/tests/postgres/test_postgres.py b/tests/postgres/test_postgres.py index efff40e8b..a093a4fdc 100644 --- a/tests/postgres/test_postgres.py +++ b/tests/postgres/test_postgres.py @@ -307,10 +307,9 @@ def test_get_form_empty_query(postgres_connector): """It should give suggestions of the databases without changing the rest""" current_config = {} form = PostgresDataSource.get_form(postgres_connector, current_config) - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'type': 'string', 'enum': ['postgres', 'postgres_db'], } @@ -320,17 +319,15 @@ def test_get_form_query_with_good_database(postgres_connector, mocker): """It should give suggestions of the collections""" current_config = {'database': 'postgres_db'} form = PostgresDataSource.get_form(postgres_connector, current_config) - assert form['properties']['database'] == {'$ref': '#/definitions/database'} - assert form['definitions']['database'] == { + assert form['properties']['database'] == {'$ref': '#/$defs/database'} + assert form['$defs']['database'] == { 'title': 'database', - 'description': 'An enumeration.', 'type': 'string', 'enum': ['postgres', 'postgres_db'], } - assert form['properties']['table'] == {'$ref': '#/definitions/table'} - assert form['definitions']['table'] == { + assert form['properties']['table'] == {'allOf': [{'$ref': '#/$defs/table'}], 'default': None} + assert form['$defs']['table'] == { 'title': 'table', - 'description': 'An enumeration.', 'type': 'string', 'enum': ['city', 'country', 'countrylanguage'], } diff --git a/tests/snowflake/test_snowflake.py b/tests/snowflake/test_snowflake.py index 27941074f..77e26ffe7 100644 --- a/tests/snowflake/test_snowflake.py +++ b/tests/snowflake/test_snowflake.py @@ -145,7 +145,9 @@ def test_datasource_get_form( snowflake_connector: SnowflakeConnector, snowflake_cursor: _SFCursor, ): - snowflake_cursor.set_side_effect([['warehouse_1', 'warehouse_2'], ['database_1', 'database_2']]) + snowflake_cursor.set_side_effect( + [{'name': ['warehouse_1', 'warehouse_2']}, {'name': ['database_1', 'database_2']}] + ) result = snowflake_datasource.get_form(snowflake_connector, {}) assert 'warehouse_1' == result['properties']['warehouse']['default'] diff --git a/tests/test_connector.py b/tests/test_connector.py index cfebcebe3..18619465c 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -4,7 +4,7 @@ import pandas as pd import pytest import tenacity as tny -from pydantic import ValidationError, create_model +from pydantic import create_model from toucan_connectors.common import ConnectorStatus from toucan_connectors.google_sheets_2.google_sheets_2_connector import GoogleSheets2Connector diff --git a/tests/test_datasource.py b/tests/test_datasource.py index a90d8f219..7ae79a1f7 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -29,19 +29,15 @@ def test_instantiation(): def test_required_arg(): # error with missing required arg data_source = {'name': 'my_name', 'collection': 'my_collection', 'query': {}} - with pytest.raises(ValidationError) as e: + with pytest.raises(ValidationError, match='Field required'): DataSource(**data_source) - assert 'domain' in e.value.errors()[0]['loc'] # Are we testing pydantic here ? - assert e.value.errors()[0]['msg'] == 'field required' def test_required_arg_wrong_type(): # error with required arg of wrong type data_source = {'domain': [], 'name': 'my_name', 'collection': 'my_collection', 'query': {}} - with pytest.raises(ValidationError) as e: + with pytest.raises(ValidationError, match='Input should be a valid string'): DataSource(**data_source) - assert 'domain' in e.value.errors()[0]['loc'] - assert e.value.errors()[0]['msg'] == 'str type expected' def test_not_required(): @@ -88,31 +84,42 @@ def test_unknown_arg(): 'query': {}, 'unk': '@', } - with pytest.raises(ValidationError) as e: + with pytest.raises(ValidationError, match='Extra inputs are not permitted'): DataSource(**data_source) - assert 'unk' in e.value.errors()[0]['loc'] - assert e.value.errors()[0]['msg'] == 'extra fields not permitted' def test_get_form(): default_form = ToucanDataSource.get_form(None, {}) assert default_form == { - 'title': 'ToucanDataSource', - 'type': 'object', + 'additionalProperties': False, 'properties': { 'domain': {'title': 'Domain', 'type': 'string'}, 'name': {'title': 'Name', 'type': 'string'}, - 'type': {'title': 'Type', 'type': 'string'}, - 'load': {'title': 'Load', 'type': 'boolean', 'default': True}, - 'live_data': {'title': 'Live Data', 'type': 'boolean', 'default': False}, - 'validation': {'title': 'Validation', 'type': 'object'}, - 'parameters': {'title': 'Parameters', 'type': 'object'}, + 'type': { + 'anyOf': [{'type': 'string'}, {'type': 'null'}], + 'default': None, + 'title': 'Type', + }, + 'load': {'default': True, 'title': 'Load', 'type': 'boolean'}, + 'live_data': {'default': False, 'title': 'Live Data', 'type': 'boolean'}, + 'validation': { + 'anyOf': [{'type': 'object'}, {'type': 'null'}], + 'default': None, + 'title': 'Validation', + }, + 'parameters': { + 'anyOf': [{'type': 'object'}, {'type': 'null'}], + 'default': None, + 'title': 'Parameters', + }, 'cache_ttl': { - 'title': "Slow Queries' Cache Expiration Time", + 'anyOf': [{'type': 'integer'}, {'type': 'null'}], + 'default': None, 'description': 'In seconds. Will override the 5min instance default and/or the connector value', - 'type': 'integer', + 'title': "Slow Queries' Cache Expiration Time", }, }, 'required': ['domain', 'name'], - 'additionalProperties': False, + 'title': 'ToucanDataSource', + 'type': 'object', } diff --git a/toucan_connectors/google_adwords/google_adwords_connector.py b/toucan_connectors/google_adwords/google_adwords_connector.py index 60e074ca0..ef137daaf 100644 --- a/toucan_connectors/google_adwords/google_adwords_connector.py +++ b/toucan_connectors/google_adwords/google_adwords_connector.py @@ -192,7 +192,7 @@ def prepare_service_query(client: AdWordsClient, data_source: GoogleAdwordsDataS ) # Build Limit if not data_source.limit: - data_source.limit = 100 + data_source.limit = '100' service_query_builder.Limit(0, int(data_source.limit)) return service, service_query_builder.Build() diff --git a/toucan_connectors/google_analytics/google_analytics_connector.py b/toucan_connectors/google_analytics/google_analytics_connector.py index 8f77d1c80..49c770986 100644 --- a/toucan_connectors/google_analytics/google_analytics_connector.py +++ b/toucan_connectors/google_analytics/google_analytics_connector.py @@ -1,5 +1,3 @@ -from typing import List - import pandas as pd from apiclient.discovery import build from oauth2client.service_account import ServiceAccountCredentials @@ -16,20 +14,20 @@ class Dimension(BaseModel): name: str - histogramBuckets: List[str] = None + histogramBuckets: list[str] | None = None class DimensionFilter(BaseModel): dimensionName: str operator: str - expressions: List[str] = None + expressions: list[str] | None = None caseSensitive: bool = False model_config = ConfigDict(extra='allow') class DimensionFilterClause(BaseModel): operator: str - filters: List[DimensionFilter] + filters: list[DimensionFilter] class DateRange(BaseModel): @@ -39,7 +37,7 @@ class DateRange(BaseModel): class Metric(BaseModel): expression: str - alias: str = None + alias: str | None = None model_config = ConfigDict(extra='allow') @@ -52,52 +50,52 @@ class MetricFilter(BaseModel): class MetricFilterClause(BaseModel): operator: str - filters: List[MetricFilter] + filters: list[MetricFilter] class OrderBy(BaseModel): fieldName: str - orderType: str = None - sortOrder: str = None + orderType: str | None = None + sortOrder: str | None = None class Pivot(BaseModel): - dimensions: List[Dimension] = None - dimensionFilterClauses: List[DimensionFilterClause] = None - metrics: List[Metric] = None - startGroup: int = None - maxGroupCount: int = None + dimensions: list[Dimension] | None = None + dimensionFilterClauses: list[DimensionFilterClause] | None = None + metrics: list[Metric] | None = None + startGroup: int | None = None + maxGroupCount: int | None = None class Cohort(BaseModel): name: str type: str - dateRange: DateRange = None + dateRange: DateRange | None = None class CohortGroup(BaseModel): - cohorts: List[Cohort] + cohorts: list[Cohort] lifetimeValue: bool = False class Segment(BaseModel): - segmentId: str = None + segmentId: str | None = None # TODO dynamicSegment: DynamicSegment class ReportRequest(BaseModel): viewId: str - dateRanges: List[DateRange] = None - samplingLevel: str = None - dimensions: List[Dimension] = None - dimensionFilterClauses: List[DimensionFilterClause] = None - metrics: List[Metric] = None - metricFilterClauses: List[MetricFilterClause] = None + dateRanges: list[DateRange] | None = None + samplingLevel: str | None = None + dimensions: list[Dimension] | None = None + dimensionFilterClauses: list[DimensionFilterClause] | None = None + metrics: list[Metric] | None = None + metricFilterClauses: list[MetricFilterClause] | None = None filtersExpression: str = '' - orderBys: List[OrderBy] = [] - segments: List[Segment] = [] - pivots: List[Pivot] = None - cohortGroup: CohortGroup = None + orderBys: list[OrderBy] = [] + segments: list[Segment] = [] + pivots: list[Pivot] | None = None + cohortGroup: CohortGroup | None = None pageToken: str = '' pageSize: int = 10000 includeEmptyRows: bool = False @@ -168,7 +166,7 @@ class GoogleAnalyticsConnector(ToucanConnector, data_source_model=GoogleAnalytic 'You should use "service_account" credentials, which is the preferred type of credentials ' 'to use when authenticating on behalf of a service or application', ) - scope: List[str] = Field( + scope: list[str] = Field( [SCOPE], description='OAuth 2.0 scopes define the level of access you need to ' 'request the Google APIs. For more information, see this ' diff --git a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py index b9d732357..4d2e6baf2 100644 --- a/toucan_connectors/google_sheets_2/google_sheets_2_connector.py +++ b/toucan_connectors/google_sheets_2/google_sheets_2_connector.py @@ -5,7 +5,7 @@ import os from contextlib import suppress from pathlib import Path -from typing import Any, List, Optional +from typing import Any, List import pandas as pd from aiohttp import ClientSession @@ -56,13 +56,11 @@ class GoogleSheets2DataSource(ToucanDataSource): description='Can be found in your URL: ' 'https://docs.google.com/spreadsheets/d//...', ) - sheet: Optional[str] = Field( - None, title='Sheet title', description='Title of the desired sheet' - ) + sheet: str | None = Field(None, title='Sheet title', description='Title of the desired sheet') header_row: int = Field( 0, title='Header row', description='Row of the header of the spreadsheet' ) - rows_limit: int = Field( + rows_limit: int | None = Field( None, title='Rows limit', description='Maximum number of rows to retrieve' ) parameters: dict = Field(None, description='Additional URL parameters') @@ -109,7 +107,7 @@ class GoogleSheets2Connector(ToucanConnector, data_source_model=GoogleSheets2Dat _auth_flow = 'oauth2' _oauth_trigger = 'instance' oauth2_version: str = Field('1', **{'ui.hidden': True}) - auth_flow_id: Optional[str] = None + auth_flow_id: str | None = None # TODO: turn into a class property _baseroute = 'https://sheets.googleapis.com/v4/spreadsheets/' @@ -232,10 +230,10 @@ def get_status(self) -> ConnectorStatus: def get_slice( self, data_source: GoogleSheets2DataSource, - permissions: Optional[dict] = None, + permissions: dict | None = None, offset: int = 0, limit=50, - get_row_count: Optional[bool] = False, + get_row_count: bool | None = False, ) -> DataSlice: """ Method to retrieve a part of the data as a pandas dataframe diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index 557a1098e..6892cc79b 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -149,7 +149,7 @@ def do_request(self, query, session): xpath = query['xpath'] available_params = ['url', 'method', 'params', 'data', 'json', 'headers', 'proxies'] query = {k: v for k, v in query.items() if k in available_params} - query['url'] = '/'.join([self.baseroute.rstrip('/'), query['url'].lstrip('/')]) + query['url'] = '/'.join([str(self.baseroute).rstrip('/'), query['url'].lstrip('/')]) if self.cert: # `cert` is a list of PosixPath. `request` needs a list of strings for certificates diff --git a/toucan_connectors/linkedinads/linkedinads_connector.py b/toucan_connectors/linkedinads/linkedinads_connector.py index 19c61fbb3..7dd165b2d 100644 --- a/toucan_connectors/linkedinads/linkedinads_connector.py +++ b/toucan_connectors/linkedinads/linkedinads_connector.py @@ -57,7 +57,7 @@ class LinkedinadsDataSource(ToucanDataSource): start_date: str = Field( ..., title='Start date', description='Start date of the dataset. Format must be dd/mm/yyyy.' ) - end_date: str = Field( + end_date: str | None = Field( None, title='End date', description='End date of the dataset, optional & default to today. Format must be dd/mm/yyyy.', @@ -67,9 +67,9 @@ class LinkedinadsDataSource(ToucanDataSource): title='Time granularity', description='Granularity of the dataset, default all result grouped', ) - flatten_column: str = Field(None, description='Column containing nested rows') + flatten_column: str | None = Field(None, description='Column containing nested rows') - parameters: dict = Field( + parameters: dict | None = Field( None, description='See https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting for more information', ) diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 61067207a..9f6d955bd 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -261,7 +261,7 @@ def get_slice( get_row_count: Optional[bool] = False, ) -> DataSlice: # Create a copy in order to keep the original (deepcopy-like) - data_source = MongoDataSource.parse_obj(data_source) + data_source = data_source.model_copy(deep=True) if offset or limit is not None: data_source.query = apply_condition_filter(data_source.query, permissions) data_source.query = normalize_query(data_source.query, data_source.parameters) @@ -315,7 +315,7 @@ def get_slice_with_regex( offset: Optional[int] = None, ) -> DataSlice: # Create a copy in order to keep the original (deepcopy-like) - data_source = MongoDataSource.parse_obj(data_source) + data_source = data_source.model_copy(deep=True) data_source.query = normalize_query(data_source.query, data_source.parameters) # We simply append the match regex at the end of the query, # Mongo will then optimize the pipeline to move the match regex to its most convenient position @@ -385,10 +385,9 @@ def get_unique_identifier(self) -> str: def _get_unique_datasource_identifier(self, data_source: MongoDataSource) -> dict: # let's make a copy first - data_source_rendered = MongoDataSource.parse_obj(data_source) + data_source_rendered = data_source.model_copy(deep=True) data_source_rendered.query = normalize_query(data_source.query, data_source.parameters) - del data_source_rendered.parameters - return data_source_rendered.dict() + return data_source_rendered.model_dump(exclude={'parameters'}) def get_engine_version(self) -> tuple: client = pymongo.MongoClient(**self._get_mongo_client_kwargs()) diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index 5f8e322c0..40d67e551 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -48,17 +48,17 @@ class MySQLDataSource(ToucanDataSource): **{'ui.hidden': True}, description='Deprecated, kept for compatibility purpose with old data sources configs', ) # Deprecated - table: str = Field( + table: str | None = Field( None, **{'ui.hidden': True} ) # To avoid previous config migrations, won't be used - query: Annotated[str, StringConstraints(min_length=1)] = Field( + query: Annotated[str, StringConstraints(min_length=1)] | None = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' 'the "table" parameter', widget='sql', ) - query_object: dict = Field( + query_object: dict | None = Field( None, description='An object describing a simple select query' 'This field is used internally', **{'ui.hidden': True}, diff --git a/toucan_connectors/odata/odata_connector.py b/toucan_connectors/odata/odata_connector.py index 53006b110..93b9455c4 100644 --- a/toucan_connectors/odata/odata_connector.py +++ b/toucan_connectors/odata/odata_connector.py @@ -38,7 +38,7 @@ class ODataDataSource(ToucanDataSource): class ODataConnector(ToucanConnector, data_source_model=ODataDataSource): baseroute: HttpUrl = Field(..., title='API endpoint', description='Baseroute URL') - auth: Auth = Field(None, title='Authentication type') + auth: Auth | None = Field(None, title='Authentication type') def _retrieve_data(self, data_source: ODataDataSource) -> pd.DataFrame: if self.auth: @@ -46,7 +46,7 @@ def _retrieve_data(self, data_source: ODataDataSource) -> pd.DataFrame: else: session = None - service = ODataService(self.baseroute, reflect_entities=True, session=session) + service = ODataService(str(self.baseroute), reflect_entities=True, session=session) entities = service.entities[data_source.entity] data = service.query(entities).raw(data_source.query) return pd.DataFrame(data) diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index 928bc513b..31fba48d4 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -8,21 +8,12 @@ from abc import ABC, ABCMeta, abstractmethod from enum import Enum from functools import reduce, wraps -from typing import ( - TYPE_CHECKING, - Annotated, - Any, - Generic, - Iterable, - NamedTuple, - Type, - TypeVar, - Union, -) +from typing import Annotated, Any, Generic, Iterable, NamedTuple, Type, TypeVar, Union import pandas as pd import tenacity as tny from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, SecretStr +from pydantic.fields import ModelPrivateAttr from toucan_connectors.common import ( ConnectorStatus, @@ -39,9 +30,6 @@ except ImportError: pass -if TYPE_CHECKING: - from pydantic.main import IncEx - LOGGER = logging.getLogger(__name__) @@ -121,7 +109,7 @@ def get_form(cls, connector: C, current_config): By default, we simply return the model schema. """ - return cls.schema() + return cls.model_json_schema() class RetryPolicy(BaseModel): @@ -231,10 +219,14 @@ def get_oauth2_configuration(cls): """Return a tuple indicating if the connector is an oauth2 connector and in this case, where can the credentials be located """ - oauth2_enabled = hasattr(cls, '_auth_flow') and 'oauth2' in getattr(cls, '_auth_flow') + oauth2_enabled = False + if hasattr(cls, '_auth_flow'): + assert isinstance(cls._auth_flow, ModelPrivateAttr) + oauth2_enabled = 'oauth2' in cls._auth_flow.get_default() oauth2_credentials_location = None if hasattr(cls, '_oauth_trigger'): - oauth2_credentials_location = getattr(cls, '_oauth_trigger') + assert isinstance(cls._oauth_trigger, ModelPrivateAttr) + oauth2_credentials_location = cls._oauth_trigger.get_default() return oauth2_enabled, oauth2_credentials_location @@ -271,9 +263,8 @@ def get_connector_secrets_form(cls) -> ConnectorSecretsForm | None: DS = TypeVar('DS', bound=ToucanDataSource) - PlainJsonSecretStr = Annotated[ - SecretStr, PlainSerializer(SecretStr.get_secret_value, return_type=str) + SecretStr, PlainSerializer(SecretStr.get_secret_value, return_type=str, when_used='json') ] @@ -471,7 +462,6 @@ def get_cache_key( 'offset': offset, 'limit': limit, } - if data_source is not None: unique_identifier['datasource'] = self._get_unique_datasource_identifier(data_source) json_uid = JsonWrapper.dumps(unique_identifier, sort_keys=True, default=hash) @@ -486,33 +476,6 @@ def get_identifier(self): def describe(self, data_source: DS): """ """ - # def model_dump_json( - # self, - # *, - # indent: int | None = None, - # include: 'IncEx' = None, - # exclude: 'IncEx' = None, - # by_alias: bool = False, - # exclude_unset: bool = False, - # exclude_defaults: bool = False, - # exclude_none: bool = False, - # round_trip: bool = False, - # warnings: bool = True, - # ) -> str: - # if exclude is None: - # exclude = {'data_source_model'} - # return super().model_dump_json( - # indent=indent, - # include=include, - # exclude=exclude, - # by_alias=by_alias, - # exclude_unset=exclude_unset, - # exclude_defaults=exclude_defaults, - # exclude_none=exclude_none, - # round_trip=round_trip, - # warnings=warnings, - # ) - TableInfo = dict[str, Union[str, list[dict[str, str]]]] From 8fa9d9a5767b3c83d40bfe79ec35b8c93266f85b Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 12 Sep 2023 15:36:52 +0200 Subject: [PATCH 14/27] blindly try to fix oracle tests Signed-off-by: Luka Peschke --- tests/oracle_sql/test_oracle_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/oracle_sql/test_oracle_sql.py b/tests/oracle_sql/test_oracle_sql.py index a2f3a0b38..fc51b3fe7 100644 --- a/tests/oracle_sql/test_oracle_sql.py +++ b/tests/oracle_sql/test_oracle_sql.py @@ -102,7 +102,7 @@ def test_get_form_empty_query(oracle_connector): """It should give suggestions of the databases without changing the rest""" current_config = {} form = OracleSQLDataSource.get_form(oracle_connector, current_config) - assert 'CITY' in form['definitions']['table']['enum'] + assert 'CITY' in form['$defs']['table']['enum'] assert form['required'] == ['domain', 'name'] From e47c4773f7812dcf0ada3f2fe1e682475482f444 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 15 Sep 2023 11:06:50 +0200 Subject: [PATCH 15/27] adapt recently merged code Signed-off-by: Luka Peschke --- README.md | 4 +--- templates/connector.py.m4 | 4 +--- toucan_connectors/dataiku/dataiku_connector.py | 2 -- toucan_connectors/s3/s3_connector.py | 3 +-- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b0e9c6c2e..8e76c88a6 100644 --- a/README.md +++ b/README.md @@ -155,10 +155,8 @@ class MyTypeDataSource(ToucanDataSource): query: str -class MyTypeConnector(ToucanConnector): +class MyTypeConnector(ToucanConnector,data_source_model=MyTypeDataSource): """Model of my connector""" - data_source_model: MyTypeDataSource - host: str port: int database: str diff --git a/templates/connector.py.m4 b/templates/connector.py.m4 index 8f45a3452..12942974a 100644 --- a/templates/connector.py.m4 +++ b/templates/connector.py.m4 @@ -8,9 +8,7 @@ class cap(TYPE)DataSource(ToucanDataSource): query: str -class cap(TYPE)Connector(ToucanConnector): - data_source_model: cap(TYPE)DataSource - +class cap(TYPE)Connector(ToucanConnector, data_source_model=cap(TYPE)DataSource): username: str password: str diff --git a/toucan_connectors/dataiku/dataiku_connector.py b/toucan_connectors/dataiku/dataiku_connector.py index cf5f5cc0b..af460ce78 100644 --- a/toucan_connectors/dataiku/dataiku_connector.py +++ b/toucan_connectors/dataiku/dataiku_connector.py @@ -17,8 +17,6 @@ class DataikuConnector(ToucanConnector, data_source_model=DataikuDataSource): [DSS API](https://doc.dataiku.com/dss/2.0/api/index.html). """ - data_source_model: type[DataikuDataSource] = DataikuDataSource - host: str = Field( ..., description='The domain name (preferred option as more dynamic) or ' diff --git a/toucan_connectors/s3/s3_connector.py b/toucan_connectors/s3/s3_connector.py index 47f017044..b5fd0a723 100644 --- a/toucan_connectors/s3/s3_connector.py +++ b/toucan_connectors/s3/s3_connector.py @@ -23,8 +23,7 @@ class S3DataSource(ToucanDataSource): fetcher_kwargs: dict[str, Any] = {} -class S3Connector(ToucanConnector): - data_source_model: S3DataSource +class S3Connector(ToucanConnector, data_source_model=S3DataSource): _sts_role: dict bucket_name: str = Field(..., description='Your Bucket Name') From 17449228b3aca6571b342b521816a1b22f6ce16a Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 29 Sep 2023 09:56:50 +0200 Subject: [PATCH 16/27] Update tests/google_sheets_2/test_google_sheets_2.py Co-authored-by: Eric Jolibois --- tests/google_sheets_2/test_google_sheets_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/google_sheets_2/test_google_sheets_2.py b/tests/google_sheets_2/test_google_sheets_2.py index 296b7515d..cba73a257 100644 --- a/tests/google_sheets_2/test_google_sheets_2.py +++ b/tests/google_sheets_2/test_google_sheets_2.py @@ -83,7 +83,7 @@ async def test_authentified_fetch(mocker, con): def get_columns_in_schema(schema): """Pydantic generates schema slightly differently in python <=3.7 and in python 3.8""" try: - if (defs := schema.get('definitions')) or (defs := schema.get('$defs')): + if (defs := schema.get('$defs') or schema.get('definitions')): return defs['sheet']['enum'] else: return schema['properties']['sheet']['enum'] From d0dc26143b9481c443f2c98a9d9c8fe4d767555e Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 29 Sep 2023 09:55:10 +0200 Subject: [PATCH 17/27] bump pydandic ^2.4.2 Signed-off-by: Luka Peschke --- poetry.lock | 224 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 113 insertions(+), 113 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7bca638ac..e43d4c987 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2802,18 +2802,18 @@ files = [ [[package]] name = "pydantic" -version = "2.3.0" +version = "2.4.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-2.3.0-py3-none-any.whl", hash = "sha256:45b5e446c6dfaad9444819a293b921a40e1db1aa61ea08aede0522529ce90e81"}, - {file = "pydantic-2.3.0.tar.gz", hash = "sha256:1607cc106602284cd4a00882986570472f193fde9cb1259bceeaedb26aa79a6d"}, + {file = "pydantic-2.4.2-py3-none-any.whl", hash = "sha256:bc3ddf669d234f4220e6e1c4d96b061abe0998185a8d7855c0126782b7abc8c1"}, + {file = "pydantic-2.4.2.tar.gz", hash = "sha256:94f336138093a5d7f426aac732dcfe7ab4eb4da243c88f891d65deb4a2556ee7"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.6.3" +pydantic-core = "2.10.1" typing-extensions = ">=4.6.1" [package.extras] @@ -2821,117 +2821,117 @@ email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.6.3" +version = "2.10.1" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic_core-2.6.3-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:1a0ddaa723c48af27d19f27f1c73bdc615c73686d763388c8683fe34ae777bad"}, - {file = "pydantic_core-2.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5cfde4fab34dd1e3a3f7f3db38182ab6c95e4ea91cf322242ee0be5c2f7e3d2f"}, - {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5493a7027bfc6b108e17c3383959485087d5942e87eb62bbac69829eae9bc1f7"}, - {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:84e87c16f582f5c753b7f39a71bd6647255512191be2d2dbf49458c4ef024588"}, - {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:522a9c4a4d1924facce7270c84b5134c5cabcb01513213662a2e89cf28c1d309"}, - {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaafc776e5edc72b3cad1ccedb5fd869cc5c9a591f1213aa9eba31a781be9ac1"}, - {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a750a83b2728299ca12e003d73d1264ad0440f60f4fc9cee54acc489249b728"}, - {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e8b374ef41ad5c461efb7a140ce4730661aadf85958b5c6a3e9cf4e040ff4bb"}, - {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b594b64e8568cf09ee5c9501ede37066b9fc41d83d58f55b9952e32141256acd"}, - {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2a20c533cb80466c1d42a43a4521669ccad7cf2967830ac62c2c2f9cece63e7e"}, - {file = "pydantic_core-2.6.3-cp310-none-win32.whl", hash = "sha256:04fe5c0a43dec39aedba0ec9579001061d4653a9b53a1366b113aca4a3c05ca7"}, - {file = "pydantic_core-2.6.3-cp310-none-win_amd64.whl", hash = "sha256:6bf7d610ac8f0065a286002a23bcce241ea8248c71988bda538edcc90e0c39ad"}, - {file = "pydantic_core-2.6.3-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:6bcc1ad776fffe25ea5c187a028991c031a00ff92d012ca1cc4714087e575973"}, - {file = "pydantic_core-2.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:df14f6332834444b4a37685810216cc8fe1fe91f447332cd56294c984ecbff1c"}, - {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b7486d85293f7f0bbc39b34e1d8aa26210b450bbd3d245ec3d732864009819"}, - {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a892b5b1871b301ce20d40b037ffbe33d1407a39639c2b05356acfef5536d26a"}, - {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:883daa467865e5766931e07eb20f3e8152324f0adf52658f4d302242c12e2c32"}, - {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4eb77df2964b64ba190eee00b2312a1fd7a862af8918ec70fc2d6308f76ac64"}, - {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce8c84051fa292a5dc54018a40e2a1926fd17980a9422c973e3ebea017aa8da"}, - {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22134a4453bd59b7d1e895c455fe277af9d9d9fbbcb9dc3f4a97b8693e7e2c9b"}, - {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:02e1c385095efbd997311d85c6021d32369675c09bcbfff3b69d84e59dc103f6"}, - {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d79f1f2f7ebdb9b741296b69049ff44aedd95976bfee38eb4848820628a99b50"}, - {file = "pydantic_core-2.6.3-cp311-none-win32.whl", hash = "sha256:430ddd965ffd068dd70ef4e4d74f2c489c3a313adc28e829dd7262cc0d2dd1e8"}, - {file = "pydantic_core-2.6.3-cp311-none-win_amd64.whl", hash = "sha256:84f8bb34fe76c68c9d96b77c60cef093f5e660ef8e43a6cbfcd991017d375950"}, - {file = "pydantic_core-2.6.3-cp311-none-win_arm64.whl", hash = "sha256:5a2a3c9ef904dcdadb550eedf3291ec3f229431b0084666e2c2aa8ff99a103a2"}, - {file = "pydantic_core-2.6.3-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:8421cf496e746cf8d6b677502ed9a0d1e4e956586cd8b221e1312e0841c002d5"}, - {file = "pydantic_core-2.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bb128c30cf1df0ab78166ded1ecf876620fb9aac84d2413e8ea1594b588c735d"}, - {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37a822f630712817b6ecc09ccc378192ef5ff12e2c9bae97eb5968a6cdf3b862"}, - {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:240a015102a0c0cc8114f1cba6444499a8a4d0333e178bc504a5c2196defd456"}, - {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f90e5e3afb11268628c89f378f7a1ea3f2fe502a28af4192e30a6cdea1e7d5e"}, - {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:340e96c08de1069f3d022a85c2a8c63529fd88709468373b418f4cf2c949fb0e"}, - {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1480fa4682e8202b560dcdc9eeec1005f62a15742b813c88cdc01d44e85308e5"}, - {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f14546403c2a1d11a130b537dda28f07eb6c1805a43dae4617448074fd49c282"}, - {file = "pydantic_core-2.6.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a87c54e72aa2ef30189dc74427421e074ab4561cf2bf314589f6af5b37f45e6d"}, - {file = "pydantic_core-2.6.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f93255b3e4d64785554e544c1c76cd32f4a354fa79e2eeca5d16ac2e7fdd57aa"}, - {file = "pydantic_core-2.6.3-cp312-none-win32.whl", hash = "sha256:f70dc00a91311a1aea124e5f64569ea44c011b58433981313202c46bccbec0e1"}, - {file = "pydantic_core-2.6.3-cp312-none-win_amd64.whl", hash = "sha256:23470a23614c701b37252618e7851e595060a96a23016f9a084f3f92f5ed5881"}, - {file = "pydantic_core-2.6.3-cp312-none-win_arm64.whl", hash = "sha256:1ac1750df1b4339b543531ce793b8fd5c16660a95d13aecaab26b44ce11775e9"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:a53e3195f134bde03620d87a7e2b2f2046e0e5a8195e66d0f244d6d5b2f6d31b"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:f2969e8f72c6236c51f91fbb79c33821d12a811e2a94b7aa59c65f8dbdfad34a"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:672174480a85386dd2e681cadd7d951471ad0bb028ed744c895f11f9d51b9ebe"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:002d0ea50e17ed982c2d65b480bd975fc41086a5a2f9c924ef8fc54419d1dea3"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ccc13afee44b9006a73d2046068d4df96dc5b333bf3509d9a06d1b42db6d8bf"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:439a0de139556745ae53f9cc9668c6c2053444af940d3ef3ecad95b079bc9987"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d63b7545d489422d417a0cae6f9898618669608750fc5e62156957e609e728a5"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b44c42edc07a50a081672e25dfe6022554b47f91e793066a7b601ca290f71e42"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1c721bfc575d57305dd922e6a40a8fe3f762905851d694245807a351ad255c58"}, - {file = "pydantic_core-2.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5e4a2cf8c4543f37f5dc881de6c190de08096c53986381daebb56a355be5dfe6"}, - {file = "pydantic_core-2.6.3-cp37-none-win32.whl", hash = "sha256:d9b4916b21931b08096efed090327f8fe78e09ae8f5ad44e07f5c72a7eedb51b"}, - {file = "pydantic_core-2.6.3-cp37-none-win_amd64.whl", hash = "sha256:a8acc9dedd304da161eb071cc7ff1326aa5b66aadec9622b2574ad3ffe225525"}, - {file = "pydantic_core-2.6.3-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:5e9c068f36b9f396399d43bfb6defd4cc99c36215f6ff33ac8b9c14ba15bdf6b"}, - {file = "pydantic_core-2.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e61eae9b31799c32c5f9b7be906be3380e699e74b2db26c227c50a5fc7988698"}, - {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85463560c67fc65cd86153a4975d0b720b6d7725cf7ee0b2d291288433fc21b"}, - {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9616567800bdc83ce136e5847d41008a1d602213d024207b0ff6cab6753fe645"}, - {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e9b65a55bbabda7fccd3500192a79f6e474d8d36e78d1685496aad5f9dbd92c"}, - {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f468d520f47807d1eb5d27648393519655eadc578d5dd862d06873cce04c4d1b"}, - {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9680dd23055dd874173a3a63a44e7f5a13885a4cfd7e84814be71be24fba83db"}, - {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a718d56c4d55efcfc63f680f207c9f19c8376e5a8a67773535e6f7e80e93170"}, - {file = "pydantic_core-2.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8ecbac050856eb6c3046dea655b39216597e373aa8e50e134c0e202f9c47efec"}, - {file = "pydantic_core-2.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:788be9844a6e5c4612b74512a76b2153f1877cd845410d756841f6c3420230eb"}, - {file = "pydantic_core-2.6.3-cp38-none-win32.whl", hash = "sha256:07a1aec07333bf5adebd8264047d3dc518563d92aca6f2f5b36f505132399efc"}, - {file = "pydantic_core-2.6.3-cp38-none-win_amd64.whl", hash = "sha256:621afe25cc2b3c4ba05fff53525156d5100eb35c6e5a7cf31d66cc9e1963e378"}, - {file = "pydantic_core-2.6.3-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:813aab5bfb19c98ae370952b6f7190f1e28e565909bfc219a0909db168783465"}, - {file = "pydantic_core-2.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:50555ba3cb58f9861b7a48c493636b996a617db1a72c18da4d7f16d7b1b9952b"}, - {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e20f8baedd7d987bd3f8005c146e6bcbda7cdeefc36fad50c66adb2dd2da48"}, - {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0a5d7edb76c1c57b95df719af703e796fc8e796447a1da939f97bfa8a918d60"}, - {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f06e21ad0b504658a3a9edd3d8530e8cea5723f6ea5d280e8db8efc625b47e49"}, - {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea053cefa008fda40f92aab937fb9f183cf8752e41dbc7bc68917884454c6362"}, - {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:171a4718860790f66d6c2eda1d95dd1edf64f864d2e9f9115840840cf5b5713f"}, - {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ed7ceca6aba5331ece96c0e328cd52f0dcf942b8895a1ed2642de50800b79d3"}, - {file = "pydantic_core-2.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:acafc4368b289a9f291e204d2c4c75908557d4f36bd3ae937914d4529bf62a76"}, - {file = "pydantic_core-2.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1aa712ba150d5105814e53cb141412217146fedc22621e9acff9236d77d2a5ef"}, - {file = "pydantic_core-2.6.3-cp39-none-win32.whl", hash = "sha256:44b4f937b992394a2e81a5c5ce716f3dcc1237281e81b80c748b2da6dd5cf29a"}, - {file = "pydantic_core-2.6.3-cp39-none-win_amd64.whl", hash = "sha256:9b33bf9658cb29ac1a517c11e865112316d09687d767d7a0e4a63d5c640d1b17"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d7050899026e708fb185e174c63ebc2c4ee7a0c17b0a96ebc50e1f76a231c057"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:99faba727727b2e59129c59542284efebbddade4f0ae6a29c8b8d3e1f437beb7"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fa159b902d22b283b680ef52b532b29554ea2a7fc39bf354064751369e9dbd7"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:046af9cfb5384f3684eeb3f58a48698ddab8dd870b4b3f67f825353a14441418"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:930bfe73e665ebce3f0da2c6d64455098aaa67e1a00323c74dc752627879fc67"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:85cc4d105747d2aa3c5cf3e37dac50141bff779545ba59a095f4a96b0a460e70"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b25afe9d5c4f60dcbbe2b277a79be114e2e65a16598db8abee2a2dcde24f162b"}, - {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e49ce7dc9f925e1fb010fc3d555250139df61fa6e5a0a95ce356329602c11ea9"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:2dd50d6a1aef0426a1d0199190c6c43ec89812b1f409e7fe44cb0fbf6dfa733c"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6595b0d8c8711e8e1dc389d52648b923b809f68ac1c6f0baa525c6440aa0daa"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef724a059396751aef71e847178d66ad7fc3fc969a1a40c29f5aac1aa5f8784"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c8945a105f1589ce8a693753b908815e0748f6279959a4530f6742e1994dcb6"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c8c6660089a25d45333cb9db56bb9e347241a6d7509838dbbd1931d0e19dbc7f"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:692b4ff5c4e828a38716cfa92667661a39886e71136c97b7dac26edef18767f7"}, - {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f1a5d8f18877474c80b7711d870db0eeef9442691fcdb00adabfc97e183ee0b0"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3796a6152c545339d3b1652183e786df648ecdf7c4f9347e1d30e6750907f5bb"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b962700962f6e7a6bd77e5f37320cabac24b4c0f76afeac05e9f93cf0c620014"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56ea80269077003eaa59723bac1d8bacd2cd15ae30456f2890811efc1e3d4413"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c0ebbebae71ed1e385f7dfd9b74c1cff09fed24a6df43d326dd7f12339ec34"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:252851b38bad3bfda47b104ffd077d4f9604a10cb06fe09d020016a25107bf98"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6656a0ae383d8cd7cc94e91de4e526407b3726049ce8d7939049cbfa426518c8"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9140ded382a5b04a1c030b593ed9bf3088243a0a8b7fa9f071a5736498c5483"}, - {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d38bbcef58220f9c81e42c255ef0bf99735d8f11edef69ab0b499da77105158a"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:c9d469204abcca28926cbc28ce98f28e50e488767b084fb3fbdf21af11d3de26"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:48c1ed8b02ffea4d5c9c220eda27af02b8149fe58526359b3c07eb391cb353a2"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2b1bfed698fa410ab81982f681f5b1996d3d994ae8073286515ac4d165c2e7"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf9d42a71a4d7a7c1f14f629e5c30eac451a6fc81827d2beefd57d014c006c4a"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4292ca56751aebbe63a84bbfc3b5717abb09b14d4b4442cc43fd7c49a1529efd"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7dc2ce039c7290b4ef64334ec7e6ca6494de6eecc81e21cb4f73b9b39991408c"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:615a31b1629e12445c0e9fc8339b41aaa6cc60bd53bf802d5fe3d2c0cda2ae8d"}, - {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1fa1f6312fb84e8c281f32b39affe81984ccd484da6e9d65b3d18c202c666149"}, - {file = "pydantic_core-2.6.3.tar.gz", hash = "sha256:1508f37ba9e3ddc0189e6ff4e2228bd2d3c3a4641cbe8c07177162f76ed696c7"}, + {file = "pydantic_core-2.10.1-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:d64728ee14e667ba27c66314b7d880b8eeb050e58ffc5fec3b7a109f8cddbd63"}, + {file = "pydantic_core-2.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48525933fea744a3e7464c19bfede85df4aba79ce90c60b94d8b6e1eddd67096"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef337945bbd76cce390d1b2496ccf9f90b1c1242a3a7bc242ca4a9fc5993427a"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1392e0638af203cee360495fd2cfdd6054711f2db5175b6e9c3c461b76f5175"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0675ba5d22de54d07bccde38997e780044dcfa9a71aac9fd7d4d7a1d2e3e65f7"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:128552af70a64660f21cb0eb4876cbdadf1a1f9d5de820fed6421fa8de07c893"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f6e6aed5818c264412ac0598b581a002a9f050cb2637a84979859e70197aa9e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ecaac27da855b8d73f92123e5f03612b04c5632fd0a476e469dfc47cd37d6b2e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3c01c2fb081fced3bbb3da78510693dc7121bb893a1f0f5f4b48013201f362e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:92f675fefa977625105708492850bcbc1182bfc3e997f8eecb866d1927c98ae6"}, + {file = "pydantic_core-2.10.1-cp310-none-win32.whl", hash = "sha256:420a692b547736a8d8703c39ea935ab5d8f0d2573f8f123b0a294e49a73f214b"}, + {file = "pydantic_core-2.10.1-cp310-none-win_amd64.whl", hash = "sha256:0880e239827b4b5b3e2ce05e6b766a7414e5f5aedc4523be6b68cfbc7f61c5d0"}, + {file = "pydantic_core-2.10.1-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:073d4a470b195d2b2245d0343569aac7e979d3a0dcce6c7d2af6d8a920ad0bea"}, + {file = "pydantic_core-2.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:600d04a7b342363058b9190d4e929a8e2e715c5682a70cc37d5ded1e0dd370b4"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39215d809470f4c8d1881758575b2abfb80174a9e8daf8f33b1d4379357e417c"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eeb3d3d6b399ffe55f9a04e09e635554012f1980696d6b0aca3e6cf42a17a03b"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a7902bf75779bc12ccfc508bfb7a4c47063f748ea3de87135d433a4cca7a2f"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3625578b6010c65964d177626fde80cf60d7f2e297d56b925cb5cdeda6e9925a"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:caa48fc31fc7243e50188197b5f0c4228956f97b954f76da157aae7f67269ae8"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:07ec6d7d929ae9c68f716195ce15e745b3e8fa122fc67698ac6498d802ed0fa4"}, + {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e6f31a17acede6a8cd1ae2d123ce04d8cca74056c9d456075f4f6f85de055607"}, + {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d8f1ebca515a03e5654f88411420fea6380fc841d1bea08effb28184e3d4899f"}, + {file = "pydantic_core-2.10.1-cp311-none-win32.whl", hash = "sha256:6db2eb9654a85ada248afa5a6db5ff1cf0f7b16043a6b070adc4a5be68c716d6"}, + {file = "pydantic_core-2.10.1-cp311-none-win_amd64.whl", hash = "sha256:4a5be350f922430997f240d25f8219f93b0c81e15f7b30b868b2fddfc2d05f27"}, + {file = "pydantic_core-2.10.1-cp311-none-win_arm64.whl", hash = "sha256:5fdb39f67c779b183b0c853cd6b45f7db84b84e0571b3ef1c89cdb1dfc367325"}, + {file = "pydantic_core-2.10.1-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1f22a9ab44de5f082216270552aa54259db20189e68fc12484873d926426921"}, + {file = "pydantic_core-2.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8572cadbf4cfa95fb4187775b5ade2eaa93511f07947b38f4cd67cf10783b118"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db9a28c063c7c00844ae42a80203eb6d2d6bbb97070cfa00194dff40e6f545ab"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2a35baa428181cb2270a15864ec6286822d3576f2ed0f4cd7f0c1708472aff"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05560ab976012bf40f25d5225a58bfa649bb897b87192a36c6fef1ab132540d7"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6495008733c7521a89422d7a68efa0a0122c99a5861f06020ef5b1f51f9ba7c"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ac492c686defc8e6133e3a2d9eaf5261b3df26b8ae97450c1647286750b901"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8282bab177a9a3081fd3d0a0175a07a1e2bfb7fcbbd949519ea0980f8a07144d"}, + {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:aafdb89fdeb5fe165043896817eccd6434aee124d5ee9b354f92cd574ba5e78f"}, + {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f6defd966ca3b187ec6c366604e9296f585021d922e666b99c47e78738b5666c"}, + {file = "pydantic_core-2.10.1-cp312-none-win32.whl", hash = "sha256:7c4d1894fe112b0864c1fa75dffa045720a194b227bed12f4be7f6045b25209f"}, + {file = "pydantic_core-2.10.1-cp312-none-win_amd64.whl", hash = "sha256:5994985da903d0b8a08e4935c46ed8daf5be1cf217489e673910951dc533d430"}, + {file = "pydantic_core-2.10.1-cp312-none-win_arm64.whl", hash = "sha256:0d8a8adef23d86d8eceed3e32e9cca8879c7481c183f84ed1a8edc7df073af94"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:9badf8d45171d92387410b04639d73811b785b5161ecadabf056ea14d62d4ede"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:ebedb45b9feb7258fac0a268a3f6bec0a2ea4d9558f3d6f813f02ff3a6dc6698"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfe1090245c078720d250d19cb05d67e21a9cd7c257698ef139bc41cf6c27b4f"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e357571bb0efd65fd55f18db0a2fb0ed89d0bb1d41d906b138f088933ae618bb"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3dcd587b69bbf54fc04ca157c2323b8911033e827fffaecf0cafa5a892a0904"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c120c9ce3b163b985a3b966bb701114beb1da4b0468b9b236fc754783d85aa3"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15d6bca84ffc966cc9976b09a18cf9543ed4d4ecbd97e7086f9ce9327ea48891"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cabb9710f09d5d2e9e2748c3e3e20d991a4c5f96ed8f1132518f54ab2967221"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:82f55187a5bebae7d81d35b1e9aaea5e169d44819789837cdd4720d768c55d15"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1d40f55222b233e98e3921df7811c27567f0e1a4411b93d4c5c0f4ce131bc42f"}, + {file = "pydantic_core-2.10.1-cp37-none-win32.whl", hash = "sha256:14e09ff0b8fe6e46b93d36a878f6e4a3a98ba5303c76bb8e716f4878a3bee92c"}, + {file = "pydantic_core-2.10.1-cp37-none-win_amd64.whl", hash = "sha256:1396e81b83516b9d5c9e26a924fa69164156c148c717131f54f586485ac3c15e"}, + {file = "pydantic_core-2.10.1-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6835451b57c1b467b95ffb03a38bb75b52fb4dc2762bb1d9dbed8de31ea7d0fc"}, + {file = "pydantic_core-2.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b00bc4619f60c853556b35f83731bd817f989cba3e97dc792bb8c97941b8053a"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa467fd300a6f046bdb248d40cd015b21b7576c168a6bb20aa22e595c8ffcdd"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d99277877daf2efe074eae6338453a4ed54a2d93fb4678ddfe1209a0c93a2468"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa7db7558607afeccb33c0e4bf1c9a9a835e26599e76af6fe2fcea45904083a6"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aad7bd686363d1ce4ee930ad39f14e1673248373f4a9d74d2b9554f06199fb58"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:443fed67d33aa85357464f297e3d26e570267d1af6fef1c21ca50921d2976302"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:042462d8d6ba707fd3ce9649e7bf268633a41018d6a998fb5fbacb7e928a183e"}, + {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ecdbde46235f3d560b18be0cb706c8e8ad1b965e5c13bbba7450c86064e96561"}, + {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ed550ed05540c03f0e69e6d74ad58d026de61b9eaebebbaaf8873e585cbb18de"}, + {file = "pydantic_core-2.10.1-cp38-none-win32.whl", hash = "sha256:8cdbbd92154db2fec4ec973d45c565e767ddc20aa6dbaf50142676484cbff8ee"}, + {file = "pydantic_core-2.10.1-cp38-none-win_amd64.whl", hash = "sha256:9f6f3e2598604956480f6c8aa24a3384dbf6509fe995d97f6ca6103bb8c2534e"}, + {file = "pydantic_core-2.10.1-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:655f8f4c8d6a5963c9a0687793da37b9b681d9ad06f29438a3b2326d4e6b7970"}, + {file = "pydantic_core-2.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e570ffeb2170e116a5b17e83f19911020ac79d19c96f320cbfa1fa96b470185b"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64322bfa13e44c6c30c518729ef08fda6026b96d5c0be724b3c4ae4da939f875"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:485a91abe3a07c3a8d1e082ba29254eea3e2bb13cbbd4351ea4e5a21912cc9b0"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7c2b8eb9fc872e68b46eeaf835e86bccc3a58ba57d0eedc109cbb14177be531"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5cb87bdc2e5f620693148b5f8f842d293cae46c5f15a1b1bf7ceeed324a740c"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25bd966103890ccfa028841a8f30cebcf5875eeac8c4bde4fe221364c92f0c9a"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f323306d0556351735b54acbf82904fe30a27b6a7147153cbe6e19aaaa2aa429"}, + {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0c27f38dc4fbf07b358b2bc90edf35e82d1703e22ff2efa4af4ad5de1b3833e7"}, + {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f1365e032a477c1430cfe0cf2856679529a2331426f8081172c4a74186f1d595"}, + {file = "pydantic_core-2.10.1-cp39-none-win32.whl", hash = "sha256:a1c311fd06ab3b10805abb72109f01a134019739bd3286b8ae1bc2fc4e50c07a"}, + {file = "pydantic_core-2.10.1-cp39-none-win_amd64.whl", hash = "sha256:ae8a8843b11dc0b03b57b52793e391f0122e740de3df1474814c700d2622950a"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d43002441932f9a9ea5d6f9efaa2e21458221a3a4b417a14027a1d530201ef1b"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fcb83175cc4936a5425dde3356f079ae03c0802bbdf8ff82c035f8a54b333521"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:962ed72424bf1f72334e2f1e61b68f16c0e596f024ca7ac5daf229f7c26e4208"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cf5bb4dd67f20f3bbc1209ef572a259027c49e5ff694fa56bed62959b41e1f9"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e544246b859f17373bed915182ab841b80849ed9cf23f1f07b73b7c58baee5fb"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c0877239307b7e69d025b73774e88e86ce82f6ba6adf98f41069d5b0b78bd1bf"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:53df009d1e1ba40f696f8995683e067e3967101d4bb4ea6f667931b7d4a01357"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a1254357f7e4c82e77c348dabf2d55f1d14d19d91ff025004775e70a6ef40ada"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:524ff0ca3baea164d6d93a32c58ac79eca9f6cf713586fdc0adb66a8cdeab96a"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f0ac9fb8608dbc6eaf17956bf623c9119b4db7dbb511650910a82e261e6600f"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:320f14bd4542a04ab23747ff2c8a778bde727158b606e2661349557f0770711e"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:63974d168b6233b4ed6a0046296803cb13c56637a7b8106564ab575926572a55"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:417243bf599ba1f1fef2bb8c543ceb918676954734e2dcb82bf162ae9d7bd514"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dda81e5ec82485155a19d9624cfcca9be88a405e2857354e5b089c2a982144b2"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:14cfbb00959259e15d684505263d5a21732b31248a5dd4941f73a3be233865b9"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:631cb7415225954fdcc2a024119101946793e5923f6c4d73a5914d27eb3d3a05"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:bec7dd208a4182e99c5b6c501ce0b1f49de2802448d4056091f8e630b28e9a52"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:149b8a07712f45b332faee1a2258d8ef1fb4a36f88c0c17cb687f205c5dc6e7d"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d966c47f9dd73c2d32a809d2be529112d509321c5310ebf54076812e6ecd884"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7eb037106f5c6b3b0b864ad226b0b7ab58157124161d48e4b30c4a43fef8bc4b"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:154ea7c52e32dce13065dbb20a4a6f0cc012b4f667ac90d648d36b12007fa9f7"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e562617a45b5a9da5be4abe72b971d4f00bf8555eb29bb91ec2ef2be348cd132"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f23b55eb5464468f9e0e9a9935ce3ed2a870608d5f534025cd5536bca25b1402"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:e9121b4009339b0f751955baf4543a0bfd6bc3f8188f8056b1a25a2d45099934"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:0523aeb76e03f753b58be33b26540880bac5aa54422e4462404c432230543f33"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e0e2959ef5d5b8dc9ef21e1a305a21a36e254e6a34432d00c72a92fdc5ecda5"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da01bec0a26befab4898ed83b362993c844b9a607a86add78604186297eb047e"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f2e9072d71c1f6cfc79a36d4484c82823c560e6f5599c43c1ca6b5cdbd54f881"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f36a3489d9e28fe4b67be9992a23029c3cec0babc3bd9afb39f49844a8c721c5"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f64f82cc3443149292b32387086d02a6c7fb39b8781563e0ca7b8d7d9cf72bd7"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b4a6db486ac8e99ae696e09efc8b2b9fea67b63c8f88ba7a1a16c24a057a0776"}, + {file = "pydantic_core-2.10.1.tar.gz", hash = "sha256:0f8682dbdd2f67f8e1edddcbffcc29f60a6182b4901c367fc8c1c40d30bb0a82"}, ] [package.dependencies] @@ -4556,4 +4556,4 @@ toucan-toco = ["toucan-client"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "750ded24a3792e8a34f197467676eb408b89af15508baa17b6b9718ebacbc62d" +content-hash = "3265cbd220a8f29d330181238898213655c62ac6e73031d06f68b0de1e8c2522" diff --git a/pyproject.toml b/pyproject.toml index 50e10abd1..ffac6536e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ Authlib = "^1.0.1" cached-property = "^1.5.2" Jinja2 = "^3.0.3" jq = "^1.2.2" -pydantic = "^2.3.0" +pydantic = "^2.4.2" requests = "^2.28.0" tenacity = "^8.0.1" toucan-data-sdk = "^7.6.0" From 3b3eb12d6e96e12c114ae8aca3a02afc236bf615 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 29 Sep 2023 09:57:19 +0200 Subject: [PATCH 18/27] fix readme formatting Signed-off-by: Luka Peschke --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e76c88a6..87766bf44 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ class MyTypeDataSource(ToucanDataSource): query: str -class MyTypeConnector(ToucanConnector,data_source_model=MyTypeDataSource): +class MyTypeConnector(ToucanConnector, data_source_model=MyTypeDataSource): """Model of my connector""" host: str port: int From a9c0377de178b90c524349ed0caf942a2243ecea Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 29 Sep 2023 10:00:10 +0200 Subject: [PATCH 19/27] fix: remove merge conflict artifact Signed-off-by: Luka Peschke --- .../google_big_query/google_big_query_connector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/toucan_connectors/google_big_query/google_big_query_connector.py b/toucan_connectors/google_big_query/google_big_query_connector.py index 60e524535..b48ccadaf 100644 --- a/toucan_connectors/google_big_query/google_big_query_connector.py +++ b/toucan_connectors/google_big_query/google_big_query_connector.py @@ -131,8 +131,6 @@ def _define_query_param(name: str, value: Any) -> BigQueryParam: class GoogleBigQueryConnector( ToucanConnector, DiscoverableConnector, data_source_model=GoogleBigQueryDataSource ): - data_source_model: GoogleBigQueryDataSource - # for GoogleCredentials credentials: GoogleCredentials | None = Field( None, From dcb3c28956a5aad84e492e33bc2a1b7a409b8cba Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 29 Sep 2023 10:00:49 +0200 Subject: [PATCH 20/27] chore: format Signed-off-by: Luka Peschke --- tests/google_sheets_2/test_google_sheets_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/google_sheets_2/test_google_sheets_2.py b/tests/google_sheets_2/test_google_sheets_2.py index cba73a257..bfc92315d 100644 --- a/tests/google_sheets_2/test_google_sheets_2.py +++ b/tests/google_sheets_2/test_google_sheets_2.py @@ -83,7 +83,7 @@ async def test_authentified_fetch(mocker, con): def get_columns_in_schema(schema): """Pydantic generates schema slightly differently in python <=3.7 and in python 3.8""" try: - if (defs := schema.get('$defs') or schema.get('definitions')): + if defs := schema.get('$defs') or schema.get('definitions'): return defs['sheet']['enum'] else: return schema['properties']['sheet']['enum'] From eadde7a90352865ebf0fb1e897cb572d2026ace6 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Wed, 22 Nov 2023 13:47:19 +0100 Subject: [PATCH 21/27] chore: mypy Signed-off-by: Luka Peschke --- toucan_connectors/snowflake/snowflake_connector.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 5361bc56f..c78877f89 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -48,7 +48,10 @@ class SnowflakeDataSource(ToucanDataSource['SnowflakeConnector']): warehouse: str | None = Field(None, description='The name of the warehouse you want to query') query: str = Field( - ..., description='You can write your SQL query here', min_length=1, widget='sql' + ..., + description='You can write your SQL query here', + min_length=1, + widget='sql', # type:ignore[call-arg] ) # Pydantic sees **_UI_HIDDEN as the third argument (the default factory) and raises an error @@ -144,14 +147,14 @@ class SnowflakeConnector( 'It might require the region and cloud platform where your account is located, ' 'in the form of: "your_account_name.region_id.cloud_platform". See more details ' 'here.', - placeholder='your_account_name.region_id.cloud_platform', + placeholder='your_account_name.region_id.cloud_platform', # type:ignore[call-arg] ) authentication_method: AuthenticationMethod = Field( AuthenticationMethod.PLAIN, title='Authentication Method', description='The authentication mechanism that will be used to connect to your snowflake data source', - ui={'checkbox': False}, + ui={'checkbox': False}, # type:ignore[call-arg] ) user: str = Field(..., description='Your login username') @@ -171,7 +174,9 @@ class SnowflakeConnector( default_warehouse: str | None = Field( None, description='The default warehouse that shall be used for any data source' ) - category: Category = Field(Category.SNOWFLAKE, title='category', ui={'checkbox': False}) + category: Category = Field( + Category.SNOWFLAKE, title='category', ui={'checkbox': False} # type:ignore[call-arg] + ) @classmethod def model_json_schema( From edd28584a3fb22ede0e4985252fbee1a7ac2ed99 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 5 Dec 2023 18:14:21 +0100 Subject: [PATCH 22/27] fix: allow identifier to be None Signed-off-by: Luka Peschke --- toucan_connectors/toucan_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index 238d7b937..8c0c860a5 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -308,7 +308,7 @@ def __init_subclass__(cls, /, *, data_source_model: type[DS]): ) # Used to defined the connection - identifier: str = Field(None, **_UI_HIDDEN) # type:ignore[pydantic-field] + identifier: str | None = Field(None, **_UI_HIDDEN) # type:ignore[pydantic-field] model_config = ConfigDict(extra='forbid', validate_assignment=True) def bearer_oauth_get_endpoint( From 2d6f547fde9211283cec9dbcfe7e57d184470960 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Wed, 6 Dec 2023 16:12:05 +0100 Subject: [PATCH 23/27] fix: Add garbage code to mongo connector Signed-off-by: Luka Peschke --- toucan_connectors/mongo/mongo_connector.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 9f6d955bd..652087fe8 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -5,7 +5,7 @@ import pymongo from bson.son import SON from cached_property import cached_property -from pydantic import ConfigDict, Field, create_model, model_validator +from pydantic import ConfigDict, Field, create_model, field_validator, model_validator from toucan_connectors.common import ConnectorStatus, nosql_apply_parameters_to_query from toucan_connectors.json_wrapper import JsonWrapper @@ -116,6 +116,13 @@ class MongoDataSource(ToucanDataSource): 'Aggregation Pipeline in the MongoDB documentation', ) + # FIXME: This is needed for now because with we rely on empty queries being dicts. In pydantic + # v1, "[]" was coerced to {}, and we somehow rely on that cursed behaviour + @field_validator('query') + @classmethod + def _ensure_empty_query_is_dict(cls, query: dict | list) -> dict | list: + return query or {} + @classmethod def get_form(cls, connector: 'MongoConnector', current_config): """ From 6c440912cd69a8e0e75d4c27929373919045c9bf Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 8 Dec 2023 10:40:20 +0100 Subject: [PATCH 24/27] fix: do not use PlainJsonSecretStr for Oauth2 Signed-off-by: Luka Peschke --- toucan_connectors/oauth2_connector/oauth2connector.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/toucan_connectors/oauth2_connector/oauth2connector.py b/toucan_connectors/oauth2_connector/oauth2connector.py index 8b79d2320..30eb9c7b9 100644 --- a/toucan_connectors/oauth2_connector/oauth2connector.py +++ b/toucan_connectors/oauth2_connector/oauth2connector.py @@ -6,10 +6,9 @@ from authlib.common.security import generate_token from authlib.integrations.requests_client import OAuth2Session -from pydantic import BaseModel +from pydantic import BaseModel, SecretStr from toucan_connectors.json_wrapper import JsonWrapper -from toucan_connectors.toucan_connector import PlainJsonSecretStr class SecretsKeeper(ABC): @@ -28,7 +27,7 @@ def load(self, key: str, **kwargs) -> Any: class OAuth2ConnectorConfig(BaseModel): client_id: str - client_secret: PlainJsonSecretStr + client_secret: SecretStr class OAuth2Connector: From b96c32149d6da2355be882f38aece05b986f026c Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Dec 2023 16:29:57 +0100 Subject: [PATCH 25/27] fix horrible postgres typing Signed-off-by: Luka Peschke --- .../postgres/postgresql_connector.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/toucan_connectors/postgres/postgresql_connector.py b/toucan_connectors/postgres/postgresql_connector.py index b7a77e7e5..86c488b7b 100644 --- a/toucan_connectors/postgres/postgresql_connector.py +++ b/toucan_connectors/postgres/postgresql_connector.py @@ -1,5 +1,5 @@ from contextlib import suppress -from typing import Dict, List, Optional +from typing import List, Optional import psycopg2 as pgsql from pydantic import Field, StringConstraints, create_model @@ -25,19 +25,19 @@ class PostgresDataSource(ToucanDataSource): database: str = Field( DEFAULT_DATABASE, description='The name of the database you want to query' ) - query: Annotated[str, StringConstraints(min_length=1)] = Field( + query: Annotated[str | None, StringConstraints(min_length=1)] = Field( None, description='You can write a custom query against your ' 'database here. It will take precedence over ' 'the "table" parameter', widget='sql', ) - query_object: Dict = Field( + query_object: dict | None = Field( None, description='An object describing a simple select query' 'This field is used internally', **{'ui.hidden': True}, ) - table: Annotated[str, StringConstraints(min_length=1)] = Field( + table: Annotated[str | None, StringConstraints(min_length=1)] = Field( None, description='The name of the data table that you want to ' 'get (equivalent to "SELECT * FROM ' @@ -99,19 +99,21 @@ class PostgresConnector( Import data from PostgreSQL. """ - host: str = Field( + host: str | None = Field( None, description='The listening address of your database server (IP adress or hostname)' ) - port: int = Field(None, description='The listening port of your database server') + port: int | None = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') password: PlainJsonSecretStr = Field(None, description='Your login password') default_database: str = Field(DEFAULT_DATABASE, description='Your default database') - charset: str = Field(None, description='If you need to specify a specific character encoding.') - connect_timeout: int = Field( + charset: str | None = Field( + None, description='If you need to specify a specific character encoding.' + ) + connect_timeout: int | None = Field( None, title='Connection timeout', - description='You can set a connection timeout in seconds here, i.e. the maximum length of ' + description='You can set a connection timeout in seconds here, i.e. the maximal amount of ' 'time you want to wait for the server to respond. None by default', ) From a7e52df5635728b78771a6826d8ec946d375180a Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Dec 2023 17:17:42 +0100 Subject: [PATCH 26/27] fix horrible http_api typing Signed-off-by: Luka Peschke --- .../http_api/http_api_connector.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index 6892cc79b..2fe94ae86 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -1,6 +1,6 @@ import json from enum import Enum -from typing import Any, List, Union +from typing import Any, List from xml.etree.ElementTree import ParseError, fromstring, tostring import pandas as pd @@ -32,24 +32,24 @@ class Method(str, Enum): class Template(BaseModel): - headers: dict = Field( + headers: dict | None = Field( None, description='JSON object of HTTP headers to send with every HTTP request', examples=['{ "content-type": "application/xml" }'], ) - params: dict = Field( + params: dict | None = Field( None, description='JSON object of parameters to send in the query string of every HTTP request ' '(e.g. "offset" and "limit" in https://www/api-aseroute/data&offset=100&limit=50)', examples=['{ "offset": 100, "limit": 50 }'], ) - json_: dict = Field( + json_: dict | None = Field( None, alias='json', description='JSON object of parameters to send in the body of every HTTP request', examples=['{ "offset": 100, "limit": 50 }'], ) - proxies: dict = Field( + proxies: dict | None = Field( None, description='JSON object expressing a mapping of protocol or host to corresponding proxy', examples=['{"http": "foo.bar:3128", "http://host.name": "foo.bar:4012"}'], @@ -64,37 +64,37 @@ class HttpAPIDataSource(ToucanDataSource): 'For example "geo/countries"', ) method: Method = Field(Method.GET, title='HTTP Method') - headers: dict = Field( + headers: dict | None = Field( None, description='You can also setup headers in the Template section of your Connector see:
' 'https://docs.toucantoco.com/concepteur/tutorials/connectors/3-http-connector.html#template', examples=['{ "content-type": "application/xml" }'], ) - params: dict = Field( + params: dict | None = Field( None, title='URL params', description='JSON object of parameters to send in the query string of this HTTP request ' '(e.g. "offset" and "limit" in https://www/api-aseroute/data&offset=100&limit=50)', examples=['{ "offset": 100, "limit": 50 }'], ) - json_: dict = Field( + json_: dict | None = Field( None, alias='json', title='Body', description='JSON object of parameters to send in the body of every HTTP request', examples=['{ "offset": 100, "limit": 50 }'], ) - proxies: dict = Field( + proxies: dict | None = Field( None, description='JSON object expressing a mapping of protocol or host to corresponding proxy', examples=['{"http": "foo.bar:3128", "http://host.name": "foo.bar:4012"}'], ) - data: Union[str, dict] = Field( + data: str | dict | None = Field( None, description='JSON object to send in the body of the HTTP request' ) xpath: str = XpathSchema filter: str = FilterSchema - flatten_column: str = Field(None, description='Column containing nested rows') + flatten_column: str | None = Field(None, description='Column containing nested rows') @classmethod def model_json_schema( @@ -127,11 +127,11 @@ def model_json_schema( class HttpAPIConnector(ToucanConnector, data_source_model=HttpAPIDataSource): responsetype: ResponseType = Field(ResponseType.json, title='Content-type of response') baseroute: AnyHttpUrl = Field(..., title='Baseroute URL', description='Baseroute URL') - cert: List[FilePath] = Field( + cert: List[FilePath] | None = Field( None, title='Certificate', description='File path of your certificate if any' ) - auth: Auth = Field(None, title='Authentication type') - template: Template = Field( + auth: Auth | None = Field(None, title='Authentication type') + template: Template | None = Field( None, description='You can provide a custom template that will be used for every HTTP request', ) From a219852f65addcf5c86d33211abdea56675291e1 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Mon, 11 Dec 2023 17:30:09 +0100 Subject: [PATCH 27/27] fix horrible mysql typing Signed-off-by: Luka Peschke --- toucan_connectors/mysql/mysql_connector.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index 40d67e551..5690b13b4 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -111,15 +111,15 @@ class MySQLConnector( description='The domain name (preferred option as more dynamic) or ' 'the hardcoded IP address of your database server', ) - port: int = Field(None, description='The listening port of your database server') + port: int | None = Field(None, description='The listening port of your database server') user: str = Field(..., description='Your login username') - password: PlainJsonSecretStr = Field(None, description='Your login password') + password: PlainJsonSecretStr | None = Field(None, description='Your login password') charset: str = Field( 'utf8mb4', title='Charset', description='Character encoding. You should generally let the default "utf8mb4" here.', ) - connect_timeout: int = Field( + connect_timeout: int | None = Field( None, title='Connection timeout', description='You can set a connection timeout in seconds here, ' @@ -127,22 +127,22 @@ class MySQLConnector( 'for the server to respond. None by default', ) # SSL options - ssl_ca: PlainJsonSecretStr = Field( + ssl_ca: PlainJsonSecretStr | None = Field( None, description='The CA certificate content in PEM format to use to connect to the MySQL ' 'server. Equivalent of the --ssl-ca option of the MySQL client', ) - ssl_cert: PlainJsonSecretStr = Field( + ssl_cert: PlainJsonSecretStr | None = Field( None, description='The X509 certificate content in PEM format to use to connect to the MySQL ' 'server. Equivalent of the --ssl-cert option of the MySQL client', ) - ssl_key: PlainJsonSecretStr = Field( + ssl_key: PlainJsonSecretStr | None = Field( None, description='The X509 certificate key content in PEM format to use to connect to the MySQL ' 'server. Equivalent of the --ssl-key option of the MySQL client', ) - ssl_mode: SSLMode = Field( + ssl_mode: SSLMode | None = Field( None, description='SSL Mode to use to connect to the MySQL server. ' 'Equivalent of the --ssl-mode option of the MySQL client. Must be set in order to use SSL',