diff --git a/.codecov.yml b/.codecov.yml index ca81ee39226..02666df0a13 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -22,6 +22,7 @@ component_management: statuses: - type: project target: auto + threshold: 1% branches: - "!master" individual_components: diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 63623a2c521..277a131e5c2 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -347,16 +347,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/webserver.bash test_with_db 01 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-02: needs: changes @@ -392,16 +389,13 @@ jobs: run: ./ci/github/unit-testing/webserver.bash install - name: test run: ./ci/github/unit-testing/webserver.bash test_with_db 02 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-03: needs: changes @@ -437,16 +431,13 @@ jobs: run: ./ci/github/unit-testing/webserver.bash install - name: test run: ./ci/github/unit-testing/webserver.bash test_with_db 03 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-04: needs: changes @@ -482,7 +473,7 @@ jobs: run: ./ci/github/unit-testing/webserver.bash install - name: test run: ./ci/github/unit-testing/webserver.bash test_with_db 04 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -530,16 +521,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/storage.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-agent: needs: changes @@ -580,16 +568,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/agent.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-api: needs: changes @@ -625,16 +610,13 @@ jobs: run: ./ci/github/unit-testing/api.bash install - name: test run: ./ci/github/unit-testing/api.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-api-server: needs: changes @@ -676,16 +658,13 @@ jobs: - name: OAS backwards compatibility check if: ${{ !cancelled() }} run: ./ci/github/unit-testing/api-server.bash openapi-diff - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-autoscaling: needs: changes @@ -724,16 +703,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/autoscaling.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-catalog: needs: changes @@ -778,16 +754,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/catalog/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-clusters-keeper: needs: changes @@ -837,16 +810,13 @@ jobs: source .venv/bin/activate && \ pushd services/clusters-keeper && \ make test-ci-unit - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-datcore-adapter: needs: changes @@ -891,16 +861,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/datcore-adapter/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-director: needs: changes @@ -945,16 +912,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/director/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-director-v2: needs: changes @@ -999,16 +963,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/director-v2/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-aws-library: needs: changes @@ -1047,16 +1008,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/aws-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dask-task-models-library: needs: changes @@ -1095,16 +1053,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dask-task-models-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dask-sidecar: needs: changes @@ -1143,16 +1098,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dask-sidecar.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-payments: needs: changes @@ -1191,16 +1143,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/payments.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dynamic-scheduler: needs: changes @@ -1239,16 +1188,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dynamic-scheduler.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-resource-usage-tracker: needs: changes @@ -1297,16 +1243,13 @@ jobs: source .venv/bin/activate && \ pushd services/resource-usage-tracker && \ make test-ci-unit - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dynamic-sidecar: needs: changes @@ -1345,16 +1288,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dynamic-sidecar.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-efs-guardian: needs: changes @@ -1404,16 +1344,13 @@ jobs: source .venv/bin/activate && \ pushd services/efs-guardian && \ make test-ci-unit - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-python-linting: needs: changes @@ -1487,16 +1424,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/postgres-database.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-invitations: needs: changes @@ -1535,16 +1469,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/invitations.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-service-integration: needs: changes @@ -1583,16 +1514,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/service-integration.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-service-library: needs: changes @@ -1631,16 +1559,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/service-library.bash test_all - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-settings-library: needs: changes @@ -1679,16 +1604,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/settings-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-models-library: needs: changes @@ -1726,16 +1648,13 @@ jobs: run: ./ci/github/unit-testing/models-library.bash typecheck - name: test run: ./ci/github/unit-testing/models-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-common-library: needs: changes @@ -1774,7 +1693,7 @@ jobs: run: ./ci/github/unit-testing/common-library.bash typecheck - name: test run: ./ci/github/unit-testing/common-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 with: flags: unittests #optional @@ -1817,16 +1736,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/notifications-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-simcore-sdk: needs: changes @@ -1867,16 +1783,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/simcore-sdk.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-tests: # NOTE: this is a github required status check! @@ -1982,7 +1895,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/webserver.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2045,7 +1959,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/webserver.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2108,7 +2023,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/director-v2.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2180,7 +2096,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/director-v2.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2245,7 +2162,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/dynamic-sidecar.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2308,7 +2226,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/simcore-sdk.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt index 7e813c8de2e..3f7021f3ad6 100644 --- a/packages/dask-task-models-library/requirements/_base.txt +++ b/packages/dask-task-models-library/requirements/_base.txt @@ -11,21 +11,21 @@ click==8.1.7 # dask # distributed # typer -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # dask # distributed -dask==2024.9.0 +dask==2024.12.0 # via # -r requirements/_base.in # distributed -distributed==2024.9.0 +distributed==2024.12.0 # via dask -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via pydantic -fsspec==2024.9.0 +fsspec==2024.10.0 # via dask idna==3.10 # via email-validator @@ -42,7 +42,7 @@ jinja2==3.1.4 # distributed jsonschema==4.23.0 # via -r requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema locket==1.0.0 # via @@ -50,13 +50,13 @@ locket==1.0.0 # partd markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 mdurl==0.1.2 # via markdown-it-py msgpack==1.1.0 # via distributed -orjson==3.10.7 +orjson==3.10.12 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -68,13 +68,13 @@ orjson==3.10.7 # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.1 +packaging==24.2 # via # dask # distributed partd==1.4.2 # via dask -psutil==6.0.0 +psutil==6.1.0 # via distributed pydantic==2.10.3 # via @@ -94,7 +94,7 @@ pydantic==2.10.3 # pydantic-settings pydantic-core==2.27.1 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.0 # via # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -124,37 +124,38 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -rich==13.8.1 +rich==13.9.4 # via # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.20.0 +rpds-py==0.22.3 # via # jsonschema # referencing shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sortedcontainers==2.4.0 # via distributed tblib==3.0.0 # via distributed -toolz==0.12.1 +toolz==1.0.0 # via # dask # distributed # partd -tornado==6.4.1 +tornado==6.4.2 # via distributed -typer==0.12.5 +typer==0.15.1 # via -r requirements/../../../packages/settings-library/requirements/_base.in -types-python-dateutil==2.9.0.20240906 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # pydantic # pydantic-core + # pydantic-extra-types # typer urllib3==2.2.3 # via @@ -167,5 +168,5 @@ urllib3==2.2.3 # distributed zict==3.0.0 # via distributed -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/packages/dask-task-models-library/requirements/_test.txt b/packages/dask-task-models-library/requirements/_test.txt index b0593212939..21f3ea3ccc4 100644 --- a/packages/dask-task-models-library/requirements/_test.txt +++ b/packages/dask-task-models-library/requirements/_test.txt @@ -1,31 +1,31 @@ -appdirs==1.4.4 - # via pint -coverage==7.6.1 +coverage==7.6.8 # via # -r requirements/_test.in # pytest-cov -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in flexcache==0.3 # via pint -flexparser==0.3.1 +flexparser==0.4 # via pint icdiff==2.0.7 # via pytest-icdiff iniconfig==2.0.0 # via pytest -packaging==24.1 +packaging==24.2 # via # -c requirements/_base.txt # pytest # pytest-sugar -pint==0.24.3 +pint==0.24.4 # via -r requirements/_test.in +platformdirs==4.3.6 + # via pint pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -38,7 +38,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-icdiff==0.9 # via -r requirements/_test.in @@ -59,15 +59,16 @@ pyyaml==6.0.2 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in -six==1.16.0 +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar typing-extensions==4.12.2 # via # -c requirements/_base.txt + # faker # flexcache # flexparser # pint diff --git a/packages/dask-task-models-library/requirements/_tools.txt b/packages/dask-task-models-library/requirements/_tools.txt index 2b96415451e..7bd7ea01f5a 100644 --- a/packages/dask-task-models-library/requirements/_tools.txt +++ b/packages/dask-task-models-library/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -13,13 +13,13 @@ click==8.1.7 # -c requirements/_base.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -27,7 +27,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -35,7 +35,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.1 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -43,20 +43,21 @@ packaging==24.1 # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt platformdirs==4.3.6 # via + # -c requirements/_test.txt # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -66,9 +67,9 @@ pyyaml==6.0.2 # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit -ruff==0.6.7 +ruff==0.8.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.1.0 +setuptools==75.6.0 # via pip-tools tomlkit==0.13.2 # via pylint @@ -77,7 +78,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py index 54019faca11..03c63c07d34 100644 --- a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py +++ b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py @@ -72,6 +72,7 @@ async def dask_spec_local_cluster( asynchronous=True, name="pytest_dask_spec_local_cluster", ) as cluster: + print("Cluster dashboard link: ", cluster.dashboard_link) scheduler_address = URL(cluster.scheduler_address) monkeypatch.setenv( "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL", diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index dec4d478b86..e0a298107ed 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -131,12 +131,12 @@ click==8.1.7 # distributed # typer # uvicorn -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in @@ -147,7 +147,7 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -170,7 +170,7 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -220,7 +220,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -272,7 +272,7 @@ locket==1.0.0 # partd markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # jinja2 @@ -286,7 +286,7 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -303,19 +303,19 @@ opentelemetry-api==1.26.0 # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore @@ -324,41 +324,42 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.49b2 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-botocore==0.47b0 +opentelemetry-instrumentation-botocore==0.49b2 # via -r requirements/../../../packages/aws-library/requirements/_base.in -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-httpx==0.47b0 +opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-logging==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-propagator-aws-xray==1.0.1 # via opentelemetry-instrumentation-botocore -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi @@ -366,7 +367,7 @@ opentelemetry-semantic-conventions==0.47b0 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -414,12 +415,13 @@ orjson==3.10.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 +packaging==24.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq partd==1.4.2 @@ -432,11 +434,11 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +protobuf==5.29.0 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -537,7 +539,7 @@ python-dateutil==2.9.0.post0 # botocore python-dotenv==1.0.1 # via pydantic-settings -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -619,8 +621,6 @@ rpds-py==0.18.1 # referencing s3transfer==0.10.1 # via boto3 -setuptools==74.0.0 - # via opentelemetry-instrumentation sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 @@ -670,7 +670,7 @@ tenacity==8.5.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -678,7 +678,7 @@ toolz==0.12.1 # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed @@ -755,6 +755,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis yarl==1.9.4 # via @@ -767,7 +768,7 @@ zict==3.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # importlib-metadata diff --git a/services/autoscaling/requirements/_test.txt b/services/autoscaling/requirements/_test.txt index b6342993557..d6bdd354de7 100644 --- a/services/autoscaling/requirements/_test.txt +++ b/services/autoscaling/requirements/_test.txt @@ -147,7 +147,7 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 # via fakeredis -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 @@ -164,7 +164,7 @@ openapi-spec-validator==0.7.1 # via moto orderly-set==5.2.2 # via deepdiff -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # pytest @@ -177,7 +177,7 @@ ply==3.11 # via jsonpath-ng pprintpp==0.4.0 # via pytest-icdiff -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/_base.txt # -r requirements/_test.in @@ -228,7 +228,7 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -273,9 +273,7 @@ s3transfer==0.10.1 # -c requirements/_base.txt # boto3 setuptools==74.0.0 - # via - # -c requirements/_base.txt - # moto + # via moto six==1.16.0 # via # -c requirements/_base.txt diff --git a/services/autoscaling/requirements/_tools.txt b/services/autoscaling/requirements/_tools.txt index 94d944fd4d1..b38a6583787 100644 --- a/services/autoscaling/requirements/_tools.txt +++ b/services/autoscaling/requirements/_tools.txt @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -61,7 +61,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -72,7 +72,6 @@ ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt setuptools==74.0.0 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index b5e0a2e1b12..cb76b03f0a6 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -129,12 +129,12 @@ click==8.1.7 # distributed # typer # uvicorn -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in @@ -145,7 +145,7 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -168,7 +168,7 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -218,7 +218,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -270,7 +270,7 @@ locket==1.0.0 # partd markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # jinja2 @@ -284,7 +284,7 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -301,19 +301,19 @@ opentelemetry-api==1.26.0 # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore @@ -322,41 +322,42 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.49b2 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-botocore==0.47b0 +opentelemetry-instrumentation-botocore==0.49b2 # via -r requirements/../../../packages/aws-library/requirements/_base.in -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-httpx==0.47b0 +opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-logging==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-propagator-aws-xray==1.0.1 # via opentelemetry-instrumentation-botocore -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi @@ -364,7 +365,7 @@ opentelemetry-semantic-conventions==0.47b0 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -412,12 +413,13 @@ orjson==3.10.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 +packaging==24.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq partd==1.4.2 @@ -430,11 +432,11 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +protobuf==5.29.0 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -535,7 +537,7 @@ python-dateutil==2.9.0.post0 # botocore python-dotenv==1.0.1 # via pydantic-settings -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -617,8 +619,6 @@ rpds-py==0.18.1 # referencing s3transfer==0.10.1 # via boto3 -setuptools==74.0.0 - # via opentelemetry-instrumentation sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 @@ -668,7 +668,7 @@ tenacity==8.5.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -676,7 +676,7 @@ toolz==0.12.1 # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed @@ -753,6 +753,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis yarl==1.9.4 # via @@ -765,7 +766,7 @@ zict==3.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # importlib-metadata diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 0af083a1485..155d8b2b918 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -163,7 +163,7 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 # via fakeredis -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 @@ -185,7 +185,7 @@ openapi-spec-validator==0.7.1 # via moto orderly-set==5.2.2 # via deepdiff -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # pytest @@ -197,7 +197,7 @@ pluggy==1.5.0 # via pytest ply==3.11 # via jsonpath-ng -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/_base.txt # -r requirements/_test.in @@ -242,7 +242,7 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -287,9 +287,7 @@ s3transfer==0.10.1 # -c requirements/_base.txt # boto3 setuptools==74.0.0 - # via - # -c requirements/_base.txt - # moto + # via moto six==1.16.0 # via # -c requirements/_base.txt diff --git a/services/clusters-keeper/requirements/_tools.txt b/services/clusters-keeper/requirements/_tools.txt index 94d944fd4d1..b38a6583787 100644 --- a/services/clusters-keeper/requirements/_tools.txt +++ b/services/clusters-keeper/requirements/_tools.txt @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -61,7 +61,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -72,7 +72,6 @@ ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt setuptools==74.0.0 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 0096bb1b261..20d9a1196c0 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -1,20 +1,22 @@ -aio-pika==9.4.1 +aio-pika==9.5.3 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiobotocore==2.13.0 +aiobotocore==2.15.2 # via s3fs -aiocache==0.12.2 +aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.24.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -aiohttp==3.9.5 +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.10 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -38,15 +40,15 @@ aiohttp==3.9.5 # aiodocker # fsspec # s3fs -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.7.0 # via # fast-depends # faststream @@ -56,18 +58,18 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing -blosc==1.11.1 +blosc==1.11.2 # via -r requirements/_base.in -bokeh==3.4.1 +bokeh==3.6.2 # via dask -botocore==1.34.106 +botocore==1.35.36 # via aiobotocore -certifi==2024.7.4 +certifi==2024.8.30 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -88,63 +90,65 @@ certifi==2024.7.4 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via # dask # distributed # typer -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # dask # distributed -contourpy==1.2.1 +contourpy==1.3.1 # via bokeh -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/constraints.txt # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in # -r requirements/_base.in # distributed -deprecated==1.2.14 +deprecated==1.2.15 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via dask -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic +exceptiongroup==1.2.2 + # via aio-pika fast-depends==2.4.12 # via faststream -faststream==0.5.31 +faststream==0.5.33 # via -r requirements/../../../packages/service-library/requirements/_base.in -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -r requirements/_base.in # dask # s3fs -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.0 +grpcio==1.68.1 # via opentelemetry-exporter-otlp-proto-grpc -idna==3.7 +idna==3.10 # via # anyio # email-validator # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # dask # opentelemetry-api @@ -173,12 +177,12 @@ jinja2==3.1.4 # distributed jmespath==1.0.1 # via botocore -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via jsonschema locket==1.0.0 # via @@ -188,22 +192,22 @@ lz4==4.3.3 # via -r requirements/_base.in markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 mdurl==0.1.2 # via markdown-it-py msgpack==1.1.0 # via distributed -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl -numpy==1.26.4 +numpy==2.1.3 # via # bokeh # contourpy # pandas -opentelemetry-api==1.26.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc @@ -214,45 +218,46 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.28.2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-logging==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.49b2 # via opentelemetry-instrumentation-requests -orjson==3.10.3 +orjson==3.10.12 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -284,30 +289,35 @@ orjson==3.10.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 +packaging==24.2 # via # bokeh # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq -pandas==2.2.2 +pandas==2.2.3 # via bokeh partd==1.4.2 # via dask -pillow==10.3.0 +pillow==11.0.0 # via bokeh -prometheus-client==0.20.0 +prometheus-client==0.21.1 # via -r requirements/_base.in -protobuf==4.25.4 +propcache==0.2.1 + # via + # aiohttp + # yarl +protobuf==5.29.1 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # distributed -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -350,7 +360,7 @@ pydantic==2.10.2 # pydantic-settings pydantic-core==2.27.1 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.0 # via # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -374,7 +384,7 @@ pydantic-settings==2.6.1 # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via @@ -383,9 +393,9 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via pydantic-settings -pytz==2024.1 +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -409,7 +419,7 @@ pyyaml==6.0.1 # bokeh # dask # distributed -redis==5.0.4 +redis==5.2.0 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -430,7 +440,7 @@ redis==5.0.4 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in -referencing==0.29.3 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -440,23 +450,21 @@ repro-zipfile==0.3.1 # -r requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.4 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.1 +rpds-py==0.22.3 # via # jsonschema # referencing -s3fs==2024.5.0 +s3fs==2024.10.0 # via fsspec -setuptools==74.0.0 - # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -464,37 +472,38 @@ sortedcontainers==2.4.0 # via distributed tblib==3.0.0 # via distributed -tenacity==8.5.0 +tenacity==9.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # bokeh # distributed -tqdm==4.66.4 +tqdm==4.67.1 # via -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.15.1 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # aiodebug - # aiodocker + # anyio # faststream # opentelemetry-sdk # pydantic # pydantic-core + # pydantic-extra-types # typer -tzdata==2024.1 +tzdata==2024.2 # via pandas urllib3==2.2.3 # via @@ -519,15 +528,15 @@ urllib3==2.2.3 # botocore # distributed # requests -wrapt==1.16.0 +wrapt==1.17.0 # via # aiobotocore # deprecated # opentelemetry-instrumentation # opentelemetry-instrumentation-redis -xyzservices==2024.4.0 +xyzservices==2024.9.0 # via bokeh -yarl==1.9.4 +yarl==1.18.3 # via # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika @@ -535,5 +544,5 @@ yarl==1.9.4 # aiormq zict==3.0.0 # via distributed -zipp==3.18.2 +zipp==3.21.0 # via importlib-metadata diff --git a/services/dask-sidecar/requirements/_dask-distributed.txt b/services/dask-sidecar/requirements/_dask-distributed.txt index 78a222ea415..2e37ea361da 100644 --- a/services/dask-sidecar/requirements/_dask-distributed.txt +++ b/services/dask-sidecar/requirements/_dask-distributed.txt @@ -1,4 +1,4 @@ -blosc==1.11.1 +blosc==1.11.2 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in @@ -7,25 +7,25 @@ click==8.1.7 # -c requirements/./_base.txt # dask # distributed -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/./_base.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in # distributed -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/./_base.txt # dask -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/./_base.txt # dask -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/./_base.txt # dask @@ -42,7 +42,7 @@ lz4==4.3.3 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/./_base.txt # jinja2 @@ -50,11 +50,11 @@ msgpack==1.1.0 # via # -c requirements/./_base.txt # distributed -numpy==1.26.4 +numpy==2.1.3 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in -packaging==24.0 +packaging==24.2 # via # -c requirements/./_base.txt # dask @@ -63,11 +63,11 @@ partd==1.4.2 # via # -c requirements/./_base.txt # dask -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/./_base.txt # distributed -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/./_base.txt # dask @@ -80,13 +80,13 @@ tblib==3.0.0 # via # -c requirements/./_base.txt # distributed -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/./_base.txt # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/./_base.txt # distributed @@ -98,7 +98,7 @@ zict==3.0.0 # via # -c requirements/./_base.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/./_base.txt # importlib-metadata diff --git a/services/dask-sidecar/requirements/_test.txt b/services/dask-sidecar/requirements/_test.txt index 25ef41fa488..57070471558 100644 --- a/services/dask-sidecar/requirements/_test.txt +++ b/services/dask-sidecar/requirements/_test.txt @@ -4,38 +4,38 @@ annotated-types==0.7.0 # pydantic antlr4-python3-runtime==4.13.2 # via moto -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements/_base.txt # jsonschema # referencing -aws-sam-translator==1.89.0 +aws-sam-translator==1.94.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto -blinker==1.8.2 +blinker==1.9.0 # via flask -boto3==1.34.106 +boto3==1.35.36 # via # aws-sam-translator # moto -botocore==1.34.106 +botocore==1.35.36 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -certifi==2024.7.4 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # requests cffi==1.17.1 # via cryptography -cfn-lint==1.10.3 +cfn-lint==1.20.2 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests @@ -43,11 +43,11 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.1 +coverage==7.6.8 # via # -r requirements/_test.in # pytest-cov -cryptography==43.0.1 +cryptography==44.0.0 # via # -c requirements/../../../requirements/constraints.txt # joserfc @@ -57,19 +57,19 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in -flask==3.0.3 +flask==3.1.0 # via # flask-cors # moto flask-cors==5.0.0 # via moto -graphql-core==3.2.4 +graphql-core==3.2.5 # via moto icdiff==2.0.7 # via pytest-icdiff -idna==3.7 +idna==3.10 # via # -c requirements/_base.txt # requests @@ -88,53 +88,53 @@ jmespath==1.0.1 # -c requirements/_base.txt # boto3 # botocore -joserfc==1.0.0 +joserfc==1.0.1 # via moto jsondiff==2.2.1 # via moto jsonpatch==1.33 # via cfn-lint -jsonpath-ng==1.6.1 +jsonpath-ng==1.7.0 # via moto jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -c requirements/_base.txt # aws-sam-translator + # jsonschema-spec # openapi-schema-validator # openapi-spec-validator -jsonschema-path==0.3.3 +jsonschema-spec==0.1.3 # via openapi-spec-validator -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via # -c requirements/_base.txt # jsonschema - # openapi-schema-validator lazy-object-proxy==1.10.0 # via openapi-spec-validator -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 # werkzeug -moto==5.0.15 +moto==5.0.22 # via -r requirements/_test.in mpmath==1.3.0 # via sympy -networkx==3.3 +networkx==3.4.2 # via cfn-lint -openapi-schema-validator==0.6.2 +openapi-schema-validator==0.4.3 # via openapi-spec-validator -openapi-spec-validator==0.7.1 +openapi-spec-validator==0.5.5 # via moto -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # pytest # pytest-sugar pathable==0.4.3 - # via jsonschema-path + # via jsonschema-spec pluggy==1.5.0 # via pytest ply==3.11 @@ -145,7 +145,7 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -154,13 +154,13 @@ pydantic-core==2.27.1 # via # -c requirements/_base.txt # pydantic -pyftpdlib==2.0.0 +pyftpdlib==2.0.1 # via pytest-localftpserver -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via pytest-localftpserver -pyparsing==3.1.4 +pyparsing==3.2.0 # via moto -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -174,7 +174,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-icdiff==0.9 # via -r requirements/_test.in @@ -196,53 +196,49 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # cfn-lint # jsondiff - # jsonschema-path + # jsonschema-spec # moto # responses -referencing==0.29.3 +referencing==0.35.1 # via # -c requirements/_base.txt # jsonschema - # jsonschema-path # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via cfn-lint requests==2.32.3 # via # -c requirements/_base.txt # docker - # jsonschema-path # moto # responses responses==0.25.3 # via moto rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.18.1 +rpds-py==0.22.3 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.2 +s3transfer==0.10.4 # via boto3 -setuptools==74.0.0 - # via - # -c requirements/_base.txt - # moto -six==1.16.0 +setuptools==75.6.0 + # via moto +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil # rfc3339-validator sympy==1.13.3 # via cfn-lint -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar types-aiofiles==24.1.0.20240626 # via -r requirements/_test.in @@ -251,6 +247,8 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # aws-sam-translator # cfn-lint + # faker + # jsonschema-spec # pydantic # pydantic-core urllib3==2.2.3 @@ -261,13 +259,13 @@ urllib3==2.2.3 # docker # requests # responses -werkzeug==3.0.4 +werkzeug==3.1.3 # via # flask # moto -wrapt==1.16.0 +wrapt==1.17.0 # via # -c requirements/_base.txt # aws-xray-sdk -xmltodict==0.13.0 +xmltodict==0.14.2 # via moto diff --git a/services/dask-sidecar/requirements/_tools.txt b/services/dask-sidecar/requirements/_tools.txt index 94d944fd4d1..200f2c403cd 100644 --- a/services/dask-sidecar/requirements/_tools.txt +++ b/services/dask-sidecar/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,13 +14,13 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -28,7 +28,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -44,7 +44,7 @@ packaging==24.0 # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -53,26 +53,25 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.6.7 +ruff==0.8.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.6.0 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 @@ -82,9 +81,9 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -watchdog==5.0.2 +watchdog==6.0.0 # via -r requirements/_tools.in -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 0d514a1b886..b27992058e3 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -108,7 +108,7 @@ attrs==23.2.0 # referencing bidict==0.23.1 # via python-socketio -blosc==1.11.1 +blosc==1.11.2 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt certifi==2024.2.2 # via @@ -158,12 +158,12 @@ click==8.1.7 # distributed # typer # uvicorn -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -174,7 +174,7 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -205,7 +205,7 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -272,7 +272,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -373,7 +373,7 @@ mako==1.3.5 # alembic markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # jinja2 @@ -391,9 +391,9 @@ multidict==6.0.5 # yarl networkx==3.3 # via -r requirements/_base.in -numpy==1.26.4 +numpy==2.1.3 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt -opentelemetry-api==1.27.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -411,19 +411,19 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.27.0 +opentelemetry-exporter-otlp==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.27.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.27.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.48b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi @@ -434,45 +434,46 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-aiopg==0.48b0 +opentelemetry-instrumentation-aiopg==0.49b2 # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in -opentelemetry-instrumentation-asgi==0.48b0 +opentelemetry-instrumentation-asgi==0.49b2 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-asyncpg==0.48b0 +opentelemetry-instrumentation-asyncpg==0.49b2 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in -opentelemetry-instrumentation-dbapi==0.48b0 +opentelemetry-instrumentation-dbapi==0.49b2 # via opentelemetry-instrumentation-aiopg -opentelemetry-instrumentation-fastapi==0.48b0 +opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-httpx==0.48b0 +opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-logging==0.48b0 +opentelemetry-instrumentation-logging==0.49b2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.48b0 +opentelemetry-instrumentation-redis==0.49b2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.48b0 +opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.27.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.27.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.48b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi @@ -481,7 +482,7 @@ opentelemetry-semantic-conventions==0.48b0 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.48b0 +opentelemetry-util-http==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -549,12 +550,13 @@ orjson==3.10.3 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in # fastapi -packaging==24.0 +packaging==24.2 # via # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq partd==1.4.2 @@ -569,11 +571,11 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +protobuf==5.29.0 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -715,7 +717,7 @@ python-multipart==0.0.9 # via fastapi python-socketio==5.11.2 # via -r requirements/_base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -821,8 +823,6 @@ rpds-py==0.18.1 # via # jsonschema # referencing -setuptools==74.0.0 - # via opentelemetry-instrumentation shellingham==1.5.4 # via typer simple-websocket==1.0.0 @@ -924,7 +924,7 @@ tenacity==8.5.0 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -932,7 +932,7 @@ toolz==0.12.1 # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed @@ -1059,6 +1059,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis wsproto==1.2.0 # via simple-websocket @@ -1075,7 +1076,7 @@ zict==3.0.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # importlib-metadata diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index 6194851e0c2..0f4d71204bc 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -67,7 +67,7 @@ click==8.1.7 # -c requirements/_base.txt # dask # distributed -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/_base.txt # dask @@ -76,12 +76,12 @@ contourpy==1.3.0 # via bokeh coverage==7.6.1 # via pytest-cov -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/_base.txt # -r requirements/_test.in # distributed -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/_base.txt # dask @@ -98,7 +98,7 @@ frozenlist==1.4.1 # -c requirements/_base.txt # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/_base.txt # dask @@ -128,7 +128,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/_base.txt # dask @@ -155,7 +155,7 @@ mako==1.3.5 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # alembic -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 @@ -174,14 +174,14 @@ mypy==1.12.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -numpy==1.26.4 +numpy==2.1.3 # via # -c requirements/_base.txt # bokeh # contourpy # pandas # types-networkx -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # bokeh @@ -204,7 +204,7 @@ pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/_base.txt # distributed @@ -241,7 +241,7 @@ python-dateutil==2.9.0.post0 # pandas pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -283,18 +283,18 @@ tblib==3.0.0 # via # -c requirements/_base.txt # distributed -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/_base.txt # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/_base.txt # bokeh # distributed -types-networkx==3.2.1.20240918 +types-networkx==3.4.2.20241115 # via -r requirements/_test.in types-psycopg2==2.9.21.20240819 # via -r requirements/_test.in @@ -332,7 +332,7 @@ zict==3.0.0 # via # -c requirements/_base.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/_base.txt # importlib-metadata diff --git a/services/director-v2/requirements/_tools.txt b/services/director-v2/requirements/_tools.txt index 9588b0ee960..a4dfa668c25 100644 --- a/services/director-v2/requirements/_tools.txt +++ b/services/director-v2/requirements/_tools.txt @@ -39,7 +39,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -64,7 +64,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -74,9 +74,7 @@ pyyaml==6.0.1 ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt setuptools==74.0.0 - # via - # -c requirements/_base.txt - # pip-tools + # via pip-tools tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index 96505371754..181c6c22a6d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -16,7 +16,7 @@ from copy import deepcopy from dataclasses import dataclass, field from http.client import HTTPException -from typing import Any, cast +from typing import Any, Final, cast import dask.typing import distributed @@ -99,7 +99,7 @@ } -_DASK_DEFAULT_TIMEOUT_S = 1 +_DASK_DEFAULT_TIMEOUT_S: Final[int] = 5 _UserCallbackInSepThread = Callable[[], None] @@ -263,6 +263,9 @@ def _comp_sidecar_fct( ) # NOTE: the callback is running in a secondary thread, and takes a future as arg task_future.add_done_callback(lambda _: callback()) + await distributed.Variable(job_id, client=self.backend.client).set( + task_future + ) await dask_utils.wrap_client_async_routine( self.backend.client.publish_dataset(task_future, name=job_id) @@ -450,23 +453,34 @@ def _get_pipeline_statuses( DaskSchedulerTaskState | None, task_statuses.get(job_id, "lost") ) if dask_status == "erred": - # find out if this was a cancellation - exception = await distributed.Future(job_id).exception( - timeout=_DASK_DEFAULT_TIMEOUT_S - ) - assert isinstance(exception, Exception) # nosec - - if isinstance(exception, TaskCancelledError): - running_states.append(DaskClientTaskState.ABORTED) - else: - assert exception # nosec + try: + # find out if this was a cancellation + var = distributed.Variable(job_id, client=self.backend.client) + future: distributed.Future = await var.get( + timeout=_DASK_DEFAULT_TIMEOUT_S + ) + exception = await future.exception(timeout=_DASK_DEFAULT_TIMEOUT_S) + assert isinstance(exception, Exception) # nosec + + if isinstance(exception, TaskCancelledError): + running_states.append(DaskClientTaskState.ABORTED) + else: + assert exception # nosec + _logger.warning( + "Task %s completed in error:\n%s\nTrace:\n%s", + job_id, + exception, + "".join(traceback.format_exception(exception)), + ) + running_states.append(DaskClientTaskState.ERRED) + except TimeoutError: _logger.warning( - "Task %s completed in error:\n%s\nTrace:\n%s", + "Task %s could not be retrieved from dask-scheduler, it is lost\n" + "TIP:If the task was unpublished this can happen, or if the dask-scheduler was restarted.", job_id, - exception, - "".join(traceback.format_exception(exception)), ) - running_states.append(DaskClientTaskState.ERRED) + running_states.append(DaskClientTaskState.LOST) + elif dask_status is None: running_states.append(DaskClientTaskState.LOST) else: @@ -522,13 +536,21 @@ async def get_task_result(self, job_id: str) -> TaskOutputData: async def release_task_result(self, job_id: str) -> None: _logger.debug("releasing results for %s", f"{job_id=}") try: + # NOTE: The distributed Variable holds the future of the tasks in the dask-scheduler + # Alas, deleting the variable is done asynchronously and there is no way to ensure + # the variable was effectively deleted. + # This is annoying as one can re-create the variable without error. + var = distributed.Variable(job_id, client=self.backend.client) + var.delete() # first check if the key exists await dask_utils.wrap_client_async_routine( self.backend.client.get_dataset(name=job_id) ) + await dask_utils.wrap_client_async_routine( self.backend.client.unpublish_dataset(name=job_id) ) + except KeyError: _logger.warning("Unknown task cannot be unpublished: %s", f"{job_id=}") diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 83939689808..f8e1ccd6c61 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -9,7 +9,7 @@ import traceback from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from dataclasses import dataclass -from typing import Any, NoReturn +from typing import Any, NoReturn, cast from unittest import mock from uuid import uuid4 @@ -91,7 +91,7 @@ async def _assert_wait_for_task_status( job_id: str, dask_client: DaskClient, expected_status: DaskClientTaskState, - timeout: int | None = None, + timeout: int | None = None, # noqa: ASYNC109 ): async for attempt in AsyncRetrying( reraise=True, @@ -104,24 +104,20 @@ async def _assert_wait_for_task_status( f"waiting for task to be {expected_status=}, " f"Attempt={attempt.retry_state.attempt_number}" ) - current_task_status = (await dask_client.get_tasks_status([job_id]))[0] - assert isinstance(current_task_status, DaskClientTaskState) - print(f"{current_task_status=} vs {expected_status=}") - if ( - current_task_status is DaskClientTaskState.ERRED - and expected_status - not in [ - DaskClientTaskState.ERRED, - DaskClientTaskState.LOST, - ] - ): + got = (await dask_client.get_tasks_status([job_id]))[0] + assert isinstance(got, DaskClientTaskState) + print(f"{got=} vs {expected_status=}") + if got is DaskClientTaskState.ERRED and expected_status not in [ + DaskClientTaskState.ERRED, + DaskClientTaskState.LOST, + ]: try: # we can fail fast here # this will raise and we catch the Assertion to not reraise too long await dask_client.get_task_result(job_id) except AssertionError as exc: raise RuntimeError from exc - assert current_task_status is expected_status + assert got is expected_status @pytest.fixture @@ -364,7 +360,9 @@ def fct_that_raise_cancellation_error() -> NoReturn: async def test_dask_does_not_report_base_exception_in_task(dask_client: DaskClient): def fct_that_raise_base_exception() -> NoReturn: err_msg = "task triggers a base exception, but dask does not care..." - raise BaseException(err_msg) # pylint: disable=broad-exception-raised + raise BaseException( # pylint: disable=broad-exception-raised # noqa: TRY002 + err_msg + ) future = dask_client.backend.client.submit(fct_that_raise_base_exception) # NOTE: Since asyncio.CancelledError is derived from BaseException and the worker code checks Exception only @@ -402,7 +400,7 @@ def comp_run_metadata(faker: Faker) -> RunMetadataDict: return RunMetadataDict( product_name=faker.pystr(), simcore_user_agent=faker.pystr(), - ) | faker.pydict(allowed_types=(str,)) + ) | cast(dict[str, str], faker.pydict(allowed_types=(str,))) @pytest.fixture @@ -418,6 +416,9 @@ def task_labels(comp_run_metadata: RunMetadataDict) -> ContainerLabelsDict: @pytest.fixture def hardware_info() -> HardwareInfo: + assert "json_schema_extra" in HardwareInfo.model_config + assert isinstance(HardwareInfo.model_config["json_schema_extra"], dict) + assert isinstance(HardwareInfo.model_config["json_schema_extra"]["examples"], list) return HardwareInfo.model_validate( HardwareInfo.model_config["json_schema_extra"]["examples"][0] ) @@ -476,7 +477,9 @@ def fake_sidecar_fct( assert node_params.node_requirements.ram assert "product_name" in comp_run_metadata assert "simcore_user_agent" in comp_run_metadata - assert image_params.fake_tasks[node_id].node_requirements is not None + node_requirements = image_params.fake_tasks[node_id].node_requirements + assert node_requirements + node_id_to_job_ids = await dask_client.send_computation_tasks( user_id=user_id, project_id=project_id, @@ -491,8 +494,8 @@ def fake_sidecar_fct( f"{to_simcore_runtime_docker_label_key('user-id')}": f"{user_id}", f"{to_simcore_runtime_docker_label_key('project-id')}": f"{project_id}", f"{to_simcore_runtime_docker_label_key('node-id')}": f"{node_id}", - f"{to_simcore_runtime_docker_label_key('cpu-limit')}": f"{image_params.fake_tasks[node_id].node_requirements.cpu}", - f"{to_simcore_runtime_docker_label_key('memory-limit')}": f"{image_params.fake_tasks[node_id].node_requirements.ram}", + f"{to_simcore_runtime_docker_label_key('cpu-limit')}": f"{node_requirements.cpu}", + f"{to_simcore_runtime_docker_label_key('memory-limit')}": f"{node_requirements.ram}", f"{to_simcore_runtime_docker_label_key('product-name')}": f"{comp_run_metadata['product_name']}", f"{to_simcore_runtime_docker_label_key('simcore-user-agent')}": f"{comp_run_metadata['simcore_user_agent']}", f"{to_simcore_runtime_docker_label_key('swarm-stack-name')}": "undefined-label", @@ -605,7 +608,9 @@ def fake_sidecar_fct( ) assert published_computation_task[0].node_id in image_params.fake_tasks # creating a new future shows that it is not done???? - assert not distributed.Future(published_computation_task[0].job_id).done() + assert not distributed.Future( + published_computation_task[0].job_id, client=dask_client.backend.client + ).done() # as the task is published on the dask-scheduler when sending, it shall still be published on the dask scheduler list_of_persisted_datasets = await dask_client.backend.client.list_datasets() # type: ignore @@ -628,7 +633,9 @@ def fake_sidecar_fct( assert isinstance(task_result, TaskOutputData) assert task_result.get("some_output_key") == 123 # try to create another future and this one is already done - assert distributed.Future(published_computation_task[0].job_id).done() + assert distributed.Future( + published_computation_task[0].job_id, client=dask_client.backend.client + ).done() async def test_abort_computation_tasks( @@ -1013,9 +1020,6 @@ def fake_remote_fct( assert len(published_computation_task) == 1 assert published_computation_task[0].node_id in cpu_image.fake_tasks - # let's get a dask future for the task here so dask will not remove the task from the scheduler at the end - computation_future = distributed.Future(key=published_computation_task[0].job_id) - assert computation_future await _assert_wait_for_task_status( published_computation_task[0].job_id, @@ -1034,15 +1038,11 @@ def fake_remote_fct( ) # release the task results await dask_client.release_task_result(published_computation_task[0].job_id) - # the task is still present since we hold a future here - await _assert_wait_for_task_status( - published_computation_task[0].job_id, - dask_client, - DaskClientTaskState.ERRED if fail_remote_fct else DaskClientTaskState.SUCCESS, - ) - # removing the future will let dask eventually delete the task from its memory, so its status becomes undefined - del computation_future + await asyncio.sleep( + 5 # NOTE: here we wait to be sure that the dask-scheduler properly updates its state + ) + # the task is gone, since the distributed Variable was removed above await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, @@ -1103,9 +1103,13 @@ def fake_remote_fct( assert len(published_computation_task) == 1 assert published_computation_task[0].node_id in cpu_image.fake_tasks - computation_future = distributed.Future(published_computation_task[0].job_id) + computation_future = distributed.Future( + published_computation_task[0].job_id, client=dask_client.backend.client + ) print("--> waiting for job to finish...") - await distributed.wait(computation_future, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # type: ignore + await distributed.wait( + computation_future, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS + ) assert computation_future.done() print("job finished, now checking that we received the publications...")